From 4ebf6f907a568c9825de94175f24c6fa1c303d88 Mon Sep 17 00:00:00 2001 From: Neilhamza <132557033+Neilhamza@users.noreply.github.com> Date: Mon, 6 May 2024 22:24:39 +0300 Subject: [PATCH] [RFR] update infra to use dynamic coordinates when locating elements (#135) * -add find_on_screen method that finds element on screen and returns Coordinates -modify configuration initializing in Intellij.py -modify image_locator method -modify is is_open_mta_perspective and open_mta_perspective -Update images Signed-off-by: nhamza * pre commit changes Signed-off-by: nhamza --------- Signed-off-by: nhamza --- .../intellij/mta_perspective_active.png | Bin 2206 -> 3302 bytes src/images/intellij/mta_tab.png | Bin 2525 -> 976 bytes src/models/IDE/Intellij.py | 61 +++++++----------- src/utils/ocr.py | 29 +++++++++ 4 files changed, 54 insertions(+), 36 deletions(-) diff --git a/src/images/intellij/mta_perspective_active.png b/src/images/intellij/mta_perspective_active.png index 2cb17b741da3aedab8357f45ced5a4b9e4152d3d..e3658e3a7bf4f3e2bb6f183a077a0035e6950328 100644 GIT binary patch literal 3302 zcmVou@P)ZgXgFbngSdJ^%m!Ep$a#bVG7w zVRUJ4ZXi@?ZDjyWZ*CwlF(6H0av(A=GBhADF*-3fIy5;TK}{e@Q&j4hg2Mm+3_nRk zK~#90?VNd36lc1JpQ@%=T2@&F5rG&opooA56#)t35?3@XHxj|PMV*{Pqh@Y2YQ`Kj zMrTaM#Fz=jF%z#@CW|K52tgC0?1_3KE{uXf*<=%08fd7#e}K|xYqu>5XQrNWIH&2i zsQ351%U550)wB>WAV`ahj7+yc>}mlrGBWMR4JjkjjgT8sMy4AfH>8YAH$rYm8JTW` z+>kOd-G~R?kfN0ebKyaI={kR1fhl7YRc<^QIEa8g_Lw~wSr42CmD}!AGxyezioQ&F zJ(0xiO9->ReFTDiBrokq;-o2+C`$ipZ8y3w99?YZRU@&WakI09p{eN_Ik~@UbiLa)-m211AU0@qj+V<`)u8? zp4XQ|;n>O3utm^3wtescYsY(Z?hLOJ6SGdr?Zd1YQ@F?YDY15R#?>pDap8`){IQ7J z$ngyC?S@NF6=r}`mBW$mlSs-eK+}1e+fdMF{3~qT7)SrMDgQzGYc4-WW?CBQX=$7+ z7IAj6pz?G&>1k=CXXH@aaG#&A5G}BCcEQ2g?B;8GtJ~Alag4XeK+lQsjGVw=+w*+9 z{5hWAlEhW%{-kz9MQ}_sK5o4T9vw*^MJJwGCnlYf+lN~-reNCYMX+GNsAxQrKIQU_ zW7>18kbI`lbUmtCIE$3tG_duRCkIKgL9C@QN*YfNlz&4r#u7HkI-HCRRd&%a$z~FcuRUQld2XBIZ6ZjC%A763;aQU>-1ug&G`B97aTA1B4WlYLRzMw(5k5_SCalw7S7MU z!GDi)r>UfnDy0jPr%vF|`emf}4k5_H3c<_;|6#*1EB}Gyi@N@5a2feBn_e7-ece^c zHMYdge1-}8*06f-S=7o_e?gk5D4MG&YqFzPY%G!gSk8)1@@T&2xQMvk&7VQLo=8ht z5NrsHe~s0X>^bqyDn4)UWy*}FaUq=oK~D@iG^B`=-%tkLJSH4lbvEDpQgL@JN>eLUu=z#ye)t7$YhPnT)bIE>_jK#kMkmJRVgqH>B94=%GbQ6)UfpzvD~9a4 zu9UarHfjYMmxg0sdzG>#TP9DNO2m6BdHvvZf);d9&4^A!uGq@5VHT9;m7sN<$do4s zvf-t-$*eYR1!dBbUz3xT*>TKS{2nLG+;Q(wN%rpLJiA~COBTM$SEbHGKGC22eXH1< zREng|;Jp`~XVZ5j=+=wSBaFCtLs4dLXYuU0%ujfhRr{~vJ8c2uJ;j^;6u*7SvN`j3 zZtf!9$!f;;$*}~OFijcqXPlmnNKy|1;$CC*WP47&yP7S@g*wYBiSq_x02+QtT&8o( zA6WCx3uujuO^00QORMO_u&6+kd8uR+oF*-+3a9Xq4Crt(CSqBa#v2RcnLlqfi?(H; z7&3>rVKxXx+4aTxah{n!9EKl!nq6Be6)q&Su+W->$;i|YzGq)=0N`5l{~lf zMHaoVmGAPkI69l(VI+D&YV?shD75*ce4mHyurY+$35Y$z89l^`+~gE~L!md2ibn$J zZ&^v&w@10!0!US4e3wR9hlxX?=wucJ4Q;=w&vW3dH`tr+NzC+k7A~4eh`oT)+6F5F zyJCudg9x&|PTDtzDQ*U|HJsU>Os?QVP@qHGz$HpfWplX+kSaN!TZ-1k4qFrMgADmo z8l^`U5%+O_D#{*_w$p+pZ- z=^Q0dmU@WHN(pG8^ys%_RI3OGd<2D2c76JT3AC;z^}u0@HGovb@juUDYWy~`uXSC` zXlpNVsSLa)EM@uJr-<(J8+$js&A|(e26J))Q?3PwW&%Y&ByoD?0>(tTk*6HR0PP9h z`LP5cPXDL{3roytMBUilf!c-!Zrux!wC!`2pc+8@rZr6UR#H=1NKtVqb#+Fnw6eyE zT54+A&nMMhr&bc^*`vK+G|lbndu^LIykJJ9&C^G(5 znZHlJ@xGSOk)haR9Hpl1cXfSz`!ZUqrnVleRF-WM zw;{V8{VHoz)KgQZqfcs~S!%7Bgs!TY=CgdXZWHQRlNcHE3?pI_Xe>zL&ug}mT5jA< znn_nkGlhs$ag0M*jXd_`BIb^Hg!=SDq*iHBi28fHrKo^HT=4&$OPf=Xo{#zAdKU#| z{YNpjZv#JUnnT>YB`jb2A=$UC#Rc>xuBEhyQk?Miw!LXr_VvY6(MU;g1)aLlXoNq_ zJ?m+q;LFu4+mT62@O)mL<$vcH{thuPRjX04QFs#(>5X*p8@6rOz{U+5*tlUMyN;D% z6&y{N-HkJgID7Zy##1uE&IcbSC@s0#w)tnsu19}K5v4fd>uuMjU% zQ*|DUo|8amc_Nv6x~^tKw^1xz|6f@AxRb>13o-Zez(LS(x!TA}G$uawP?)v$d&8+@ zjuej~R&$u-Ga5jNLUD(=Kq@0;^9mYf&SrdQ7(*!}^CdD)^eW-tWuGgvG3VLWJf@&K`W59m!=zugHXl z_l*HcT!)M!q&|g%r)$sxsv*nRxhj~9bqQ=gapwmOqLl;AHd?C7uhDSZ$VAMoETOTU zraO#Uw6e!uQAbs+hKDg&L;gBnkKq`Ys`b9GjNp~MFJYZHm(|IQ^t6TQiW)RpBke}& zcTlRSy588K=)1CJ1f`0e_Et1jS5bS933+%QYc%sDa>gv;yv7rreu9D~2mHf^U{i8{ zAJ5(Wsh^~-qEz35x>hSq>box|B=t2aO;c?WhWvHD9>Xzscv@~pX|9B(a>~n1v)>I# zeT@n|wb7L|qt(rC`}Z%9>~IhMs`tpmNi`r>ZeLSfdI?AZQKa%$wS z_4&6%=z&vj71d|5_`fUa+pjc+?Eis0#3ieYjEo`V|A&;3=|;#6DI?R3kQ-7)rW+wQ kq>M~ALT*SInQp}Y0}q-`u=s}R>i_@%07*qoM6N<$f_;r*OaK4? literal 2206 zcmV;P2x0e$P)P00009a7bBm000ie z000ie0hKEb8vpGdr`}X?M5N(u-EBwJcuP!h(vjF7+;JAt4C_FhCT7G30V` zf#kgV1GijGgjo2Yv-$i4-}9dD`<(B2&UxR%fYxr<1W}^QHAJuwCCXe7ZLBEsLbS1> z%nOk>7FW9K=xarBC|G|n=GHADNLl6k2Udv3f7oA0`5l_6Dikl@@#}Z~@cVZUtyvg$ z(0nje^Gp>!kDqextlL-6}LWoG&_|{HTTXo&YV2mTb&8$pSN0pc_gqk7Ix`%Zn-jx->G#20E&|< zs7~E*fdl{mNLl6aBEx5A?%P4Iq+e;^o`7uTD=A@ffSV`BL43VP0-|<+`*)bl_;=SWo?zJ^9Bb+_;j0@`@5!5*O2h zZOsi$8jFvV?Kzq|c+)3evvi?=F>9}%uWvU`-b*~*NDKgA%rMO4O=9s0BZk1wD1CiL z{Jqn)W2NsD8LBQhO8iQ#`d8-|08rA>-KAMvYRTbui*?^uUFlB>aVBMfA+)=ur{FBf zliDBFb;-8pr`<3bd;kEXY|k-;={BQyIV&N?ZEU@KvANq$u!&PPes_T@DcD+GtXM>0 zdZ4BH+|40Kp7YF!A}czhr6q1+BKRAw9FqRej>dFa}bVKkgQjURqhgZ8`gwHVY|Pxp&|067$I$W|!u6>7|uxg?FnzI`tIdVo5s#fatl-ygp{_E3xX&Ke}hZg2Y6wZKkdQ z_RTv&;OcH4NXg5g5TImbE@(Q2Ltfg%;+$-+=C;P_1IF>_{=>&pBi!1SmhKVHsFwl2 z^poBJ+ZYD7vG<9W67j=q8Hp5G4%2Z{^~?*{GxXq=o>Isc5&*zFRo&Oye1M&Oo&8K) zydaFh1gt(BHZkbR1=(eL3$2YcHy=N@Z-Zc+jsOS@VH$q6GGj=sb6{GhHf7{x^Ck~4 z!|K*v2Lp_~>vo$xBRiE0Ck8^K=E@j#OY4w}@wqG(hZp7}bDz~hCrFn|lLg>14_jxG z5bGNY0K2=@eOn51M5@6&nMYl1!70H*2!T*QnQ5mZ?6w4(P zfiW+R?=XeAMj(WgfbO5%P%;j?OCS~^2yn&>Mkt)Y1gt(Jn;2k3rmWepnDG4QnoVYY zuug{{FobD@Ir&TDGPB=)e+vNO@BpMjoyv&;0J~l8@iCZoILTB#2S(Y&KnQ`5Pn+%j zJHUC!oH73U)WyQ}>-QcjwX|Jpyry~PopK37n8pI^=~4HTt=eh zMAFm^UMN6**UA_>IZF%!Zf!kx%x9gt70RR9t zx82gew)w~hr9Jgu=_ZDH2bybclx^F0yi9z+vkT7k< z@i_@% diff --git a/src/images/intellij/mta_tab.png b/src/images/intellij/mta_tab.png index 72d06c6ef86a72c8da5bbb031d0a741d88df3e08..a69994ba41cbf6286a03e57becd888398e11c63f 100644 GIT binary patch literal 976 zcmV;>126oEP)ZgXgFbngSdJ^%m!Ep$a#bVG7w zVRUJ4ZXi@?ZDjydXmublG$2i3av(A=GBhACGCD9cIy5vOP)#67Q&b$=3xWUu11U*F zK~z|U?U&nb97Pa@zp6gW*qgDHSO_7IP!M+rDewOPJOdI!0*OGZT;OHB_RjR7xY+R~ zIF7d00pW}NrM{Y(?yvu@s;-gmzJFyrG3oye@Q{vBtvW)r>Il`UCkf?SUNSRMO03t{ zl$;Tfrpe6==N*g1KnS7vyS-3m#x#zM;|A|MT?nWs0tm)GuLGe11_)5CQme7Ky}@cv z?7Akfy-+D7#&JWMMxG7lbX|vvAZEx6@V*Lw$PBgJ&}y~ZBC8T(+zYigrq=6gM3ra5 zIUz=nts6yzEJC?An4K^{L-F_sAZT!^tbrGZoI%;YE29BUh#1rU`ri-?YTxQ191OT-1v9o}x;9WUMziRhcU_uyT%{`ku=WAQ!Gmp$~zdL*Swszo{cl6Iv_!$X}B7 z!Q_cE{Dwu1uw?xGblstDfTOkkPPwiGyr zFJ%kV^}8THv@A4r3Dqjq44EtRoohJ7x}H7+s%;n7`8~WxRd;ik)#lQ$7s`1@t=ro^ z#E7cyg5E?SD1q?N?`y53X`;@(Rg3m!-(oRfR#~lYSj^{ReuA^cz;opbBR9y0000WhqR>|$YK9BwVefDHr+ zWhgDvX@{l_(@v)ked|pBj6QTSeaK9wCDWE6nFf+3X_?{H7_eO%5`4iIkg+7&vMpa^ z$wx=$vVAz%(peo_GEGg>`1hvW{ps89`+fWE?pbuO^$2Vn6xmQ1wrDx{^3rtWjRCh7 z01TC1KYQ^9XAf2yk#d9@jrKL}c4^mipw$X23%csv+oByE zL&2dj$!1fUQ=>`Iuu$KA;@j=6(e7type4uaX_7hbh_p7hmob!3z7=dA}L0m%s2xTCYtQXK+PIeTITicd+nF|IhGSJfQoOb zk)*VYj?UH>i)oI}p0b)Xka#Y|10YI6BT*2QN&~j(w|bSGkD>DH;CkA0OTv zTu{VEjfSi(C%?fwy7HSp#W-L}cC>%z^<5KJdKa^-szs7=`IynJ!5D`DFgH8UJ*4xw z^RED$NT3qu$`V5+r;|z~ln$y*Qkv}U5-$Ga&7I+B0!(hNIeg>Fgi?A{0RRBBR&Q0A zLywcw7Uc1lhU*C5@^$@v}oqf9-NyI9Zq7lP^n<&Zw7`Ig~F; zh)Ck}GAmHtmc2fdS^;OQKiF762|{MCuUNBU4H-%*t34KF2_iy*EMGfz{%D}AShwTX z?EnCPv9KKOy#8FdOw|O?_zoVlJiI-aBs(%t6F`W3dN*eG`Q7>#r37$tG8j!}C5jpi zObur*9w}OqEXJkjgu(5i#%^2yq=KQ+Y$6(zHC@ z-#@;j;Hga#mFzuxxJnBkB}0!pKkZF!Z*44K$D!=LQYur1g=Mj{+?7AMa3YRNTt!){ z8zl=`-926W#Wyb1ho%x>a+f>i@BBGZU`TYuK7X0r2wW^O7M@QNwNlL*005*HBEt(2 zvULTUySj-<;`A~L0KlbqWd{KzjjlcOi&Ki}$;da!oGnU5zDd5=q5uGp#!zI|1Gf}s z$>N$rdjX^W(D^faJc#9x-XKogyMFDT0p0E{(C7o-ZzpqoAa*#G4f>EQRgvG006YPVuxpLB%HtC(UnJke0IOv z2waLsIs=1JOGyFI6v`-}Mq5h;$S9!}5?O{V0kZHa1ONa4%IK^OG&Kk4hky8FD9@o6 z0Fb`4x?$I@rpB7$)U!wTZaf%W_t1+NyT75Su{q#3$Da=lKaV;m@4t8RsgeUEjoyCd zr|0}LkMF+!_P;_&iLN@mvtBV^1b0-c0Pe1k^gk=RJd#=G{(4HJbr|%n})tyTELNap=mnbDplhJgeNxe10!j@Er$4(crGC5VqAYGoy*jQ5|H%fg4|uNk{v6q;gqm&z z;0vTtW&6Kbohb^)zN@XJfaLnJ+~01mdAOFLr2V&#*hD08dYKiF)nUcl>IFK5i|cC( z5x^8U{oNy#Mg)Ku_iN{VbfLjSEI74D0#MevQ|I@?z=)x#Q8)0(wT`iba&?otsSM1; zMuy{MhYq_3uU_t$;BqRZ(i&ZJ`sXLN%Q3_tA&tp^QY?>!(GPz2@yObd%qinCaw18{ zopldb7eonqWtSD$+Ho@2_>PmoMs}Pm@QV{8OVe8d$k;3a==$@Qyq#~~3`==FYpVHH z07&1Ev2t%)+YUluDf-#%zG#+vBn>1k8DEMMG|bA!(N&N%oSN$I?g}g8mJ7l+flYE` z)YP!10PEu=Be}UWGLirQTkklza_8(j`F0!`TX!51n^o|#uztZ)(q6?ebn6`_7hmmn z95rjm?DZSwgEOp@3_b1)k=+35zI_!wA*};78tp&!#<3bL5cZa$BM!Gyei6e7 z)>vVWcHX$#-P=DJ;fhPx-VbjL%ojDizDGOMH_53-k}hxF7Q6pobaHNSaeg}3(>dYv z7cEQG(F8U+%S)E; zUmX{La+G`7sU8Vn4FLcEbj4+UpUr?$bCKs8dPoZoFNDd~6>M&KB_{bFJfv->0TH>M zGf5>4CuV1p75ar|gZ-aB8H=S>;)lhwvb++X+7scv-bcYD##48o^uwzY0JPat7ueaf zv(}Lc^*`>pdS`4d=UW6Jw|oEhPVciNgZ+K?uHGJBAaiOJN*c1bJtj)f*R&k1+tY?t zbr6QH{q}GF7w-WD;3a8_NOGxEEH;}7U=}dO4o3;2*ULXMG%l5zo|<%eyauh7!dR}7 z7!VIYA>h({baKMw_Q=f4BxA8zm&eO6j6_iaO$!t)P_#%<6h#4|7>03qys_A0FoNbQ?t4E>4?c(WH1=%&^TDF7K=4|AhEgu0E~j*NwJNkt}=`*0C#LQ zDIN!l^+h)T5V~Q60N|%#n;)>`ax zlq3@gS`c6z-{d2f95)M~>G&e6(P{-=W3d82b{b@87-KHYGYligjk4L?Hrr}10PsAI zS1e|Hb2v((lM@`rVJyiIMNyJOQ4}RfN^@Lva>C&#kq2k2F?zkr?TO9Ktd*0inFWMz nj9%~cm1VcAg_m47n@jsQrs9=)B4B}700000NkvXXu0mjft2(T| diff --git a/src/models/IDE/Intellij.py b/src/models/IDE/Intellij.py index 47869f2..11d51c7 100644 --- a/src/models/IDE/Intellij.py +++ b/src/models/IDE/Intellij.py @@ -4,9 +4,12 @@ import subprocess import time +import pyautogui + from src.models.application import Application from src.models.configuration.configurations_object import ConfigurationsObject from src.utils.general import read_file +from src.utils.ocr import find_on_screen class Intellij(Application): @@ -21,7 +24,7 @@ def __init__(self): x_coordinate=110, y_coordinate=470, ) - self.configurations = [] + self.configurations_object = ConfigurationsObject() def get_ide_version(self, ide_directory): info_file_path = os.path.join(ide_directory, "product-info.json") @@ -49,7 +52,7 @@ def delete_all_configurations(self): self.press_keys("up") self.press_keys("enter") time.sleep(1) - self.configurations = [] + self.configurations_object.configurations.clear() def create_configuration_in_ui(self): self.click(self.config_run_region) @@ -66,62 +69,48 @@ def create_configuration_in_file( config, uuid, ): - configurations_object = ConfigurationsObject() - configuration = configurations_object.create( + self.configurations_object.create( analysis_data, app_name, application_config, config, uuid, ) - self.configurations.append(configuration) - self.refresh_configuration() def image_locator(self, locator): - return f"image:{self.IMG_DIR}/intellij/{locator}" + return os.path.join(self.IMG_DIR, "intellij", locator) def is_open_mta_perspective(self): """ - Checks if MTA perspective is already opened in IntelliJ + Checks if the MTA perspective is currently visible on the screen. Returns: - (bool): True or False + bool: True if the MTA perspective is visible, False otherwise. """ - try: - self.wait_find_element(locator_type="image", locator="mta_perspective_active.png") + mta_perspective_active_path = self.image_locator("mta_perspective_active.png") + if find_on_screen(mta_perspective_active_path) is not None: return True - except Exception as exc: - try: - self.wait_find_element( - locator_type="image", - locator="mta_perspective_active_alt.png", - ) - return True - except Exception: - logging.debug( - "An error occured while finding \ - MTA perspective tab ! {}".format( - str(exc), - ), - ) - if "No matches found" in str(exc): - return False - else: - raise Exception(exc) + else: + logging.info("MTA perspective not visible on the screen.") + return False def open_mta_perspective(self): """ - Opens MTA perspective in IntelliJ + Opens MTA perspective """ if self.is_open_mta_perspective(): - logging.info("MTA perspective is already opened !") + logging.info("MTA perspective is already open!") else: - # Click on the MTA tab in left sidebar - self.click_element(locator_type="image", locator="mta_tab.png") - - self.click(self.config_run_region) + coordinates = find_on_screen(self.image_locator("mta_tab.png")) + print(coordinates) + if coordinates is None: + logging.info("MTA tab not found!") + return False + else: + pyautogui.click(coordinates) + self.click(self.config_run_region) - def run_simple_analysis(self, app_name, wait_for_analysis_finish=False): + def run_simple_analysis(self, app_name, wait_for_analysis_finish=True): self.refresh_configuration() # Search for configuration name that has to be run diff --git a/src/utils/ocr.py b/src/utils/ocr.py index 47a8e99..0f22648 100644 --- a/src/utils/ocr.py +++ b/src/utils/ocr.py @@ -1,9 +1,11 @@ +import logging import re import cv2 import numpy as np import pyautogui import pytesseract +from PIL import ImageGrab def find_all_string_occurrences(string): @@ -25,6 +27,33 @@ def find_all_string_occurrences(string): return re.findall(re.escape(string), extracted_text) +def find_on_screen(image_path, threshold=0.8): + """ + Uses OpenCV to find the element on the screen based on the provided image path. + Args: + image_path (str): The path to the image file used as the template. + threshold (float): The threshold for the match (default is 0.8). + Returns: + Coordinates x,y of the found element + """ + screen = np.array(ImageGrab.grab()) # Take a screenshot + screen_gray = cv2.cvtColor(screen, cv2.COLOR_BGR2GRAY) + template = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE) + if template is None: + logging.error(f"Template image not loaded: {image_path}") + return None + res = cv2.matchTemplate(screen_gray, template, cv2.TM_CCOEFF_NORMED) + min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res) + if max_val >= threshold: + # Calculate the center of the template match + top_left = max_loc + w, h = template.shape[::-1] + center_x = top_left[0] + w // 2 + center_y = top_left[1] + h // 2 + return (center_x, center_y) + return None + + def find_all_sentence_occurrences(sentence): screenshot = pyautogui.screenshot()