From 3179e6cabd7ce40164806b93430fa92c72ec7262 Mon Sep 17 00:00:00 2001 From: Sumit Jaiswal Date: Tue, 21 Jan 2025 18:32:04 +0530 Subject: [PATCH 1/5] code changes for aap_38439 from ols --- .../evaluation/eval_data/aap-sample.parquet | Bin 0 -> 8668 bytes scripts/evaluation/olsconfig.yaml | 62 ++++++++++++++++++ scripts/evaluation/response_evaluation.py | 9 +++ scripts/evaluation/utils/response.py | 2 - 4 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 scripts/evaluation/eval_data/aap-sample.parquet create mode 100644 scripts/evaluation/olsconfig.yaml diff --git a/scripts/evaluation/eval_data/aap-sample.parquet b/scripts/evaluation/eval_data/aap-sample.parquet new file mode 100644 index 0000000000000000000000000000000000000000..a9db0d502452fde53af76ac28a4f8b4a7d916bb9 GIT binary patch literal 8668 zcmeHNTWlNGnVykIhn&a--H0r;mAi+Er8)sciMrULV0S%~NQ$aNNfaegYz$#8hvZ1Z znekjiGl{znwh7W8aQhGx?hUDnwrGO@?L!}WL(pBI*zRtNUa-g(MN)M0un5p%u~;k? z*`fvZKZiq#l4!T-8vBwE7B%NT=lqxNznwGkI-idRJ;C+hm6aeDWZug#=RGXrXPG|# z10JUTwC|FKX??uz?;sNl1^*%ns{_GP!BfLwUwbC(W8F0&=6nw-Og?zN5576JzyH}Z z19E@S7yM7QE%{|u4#I4)CfG=^5U*K^sA6R5Dz=7}hnRDQDl|o1-?#p9`7CpZcWk{b z*ov;9I<{(&<%gNuGCshnZq%`6BU{&1>rr61q&HxI6wR^)RV94APcy>M+A#35E4o=f zZqfzje6Xh$;YUyMqB;tPy0n@0u=Zvo_|(vXS-xbx@J=o2~;(#8Wys#P`B8NUv22-KC0^`Mz~T@Bn6hKO-|5cBpm39jAX}9 z6$xO|s_e))-L$P3Dk^r(|1l=&{p)W@f`+Q5?iipz727pzA_0k}F39j?3R)E-y@K#{ z#UdQg&}SL$^k=U`k*e%t)bUT0{brz|R2|d*cQX|Q+!qD?>I50@gC4Wcr?BRTf_PYQ~LECWjln2s!LTSVKMQx~*x08w)(#X?*GB(pu2EB@S`?BB*^a42 z{R#Fr<1aGY8PgL(ADUws34iyTg$#_%aa$i}eZild2P46$b^~LLUGrG@?9EYbAj&Bk zn;Z?W*Up)MkqQATH6wEa;0QDJ3KMxDphECjd@cSZVT+yQAB``2bS?4~pXnHpF9oa^ zSExmPeO|S65N6gD+xqy$feIk|^b6+&Q>rO826N1(J~s?0V3_)W5_vgb3Wg#_h6lkH z;A>tRup~i!YQUpc*lPne6dd*j!zCUINP;0qksmX^`utmAx3r92%YI|@6-GR@e#{bL|dy4rL3RVFIKI`)^OBW*}b$t%nOP4vclk3{|C_P=cbYFD3te**X%a_`d?1UHsn|MBTEgKm9_@&6LMo2m>#cykr~UvKlT2kKCUbhZ=_ zA?)QlKgYK&^MQFTzrK1#_kLo4%W4+;Dtn~>sk#Q<+^i`T8!bSAgj#^j#HRuas_w|K zHO6Cc!Lh&n!(_V=(ALqT^uN*PV%OC=_rIqVU&R{#gSAt zD&5U!(lIHhZE;i`0xs#9EvCKjMQuPh3Nz|rXK@`bkive1arjsL?Z$VaWz(8Y?Y1#V zG1yA*v}m4@5lJ*^AW6@5vOz&~d*NZm5y**?jwT&BBG9ZOyit#?+R8z~5Q}v=yCe_MSt*neJjB} zddmDQYrhF=YPM}y^Ai)YE?F_?@ZiH3)R+nA5Dz`+kf5Y-H~~|;Ft;nT&V;)ZH}Csz z?~*eGIl@f<1QUi_nSd5eVAyqG*d;LRY6%8G`s_kM+~rSS_3%#cs%KsGUmYBWedC$j z4x`O75WM4Cn=#-5RpGtC(BT;u4uo&Ach{Au|HqW2cKnIz9;jr6lQ^)x5JI7RrKtF2TNLY^CN1pEsz6$pdAC1vx*vB4%6^P#|tgZ5e>{<>j ztmO)+Tp=+3KIYlg)aKF<%=5X;Y;q+9U2Z$IvbMeo^Qf@4wz4_T44pl@c4>WuFQnJ< zt0EbuEUetd<`Eposroy~0JDPst@!C?4EzzyA%Z}_1eDuE*0xt1w?{h#ayTPB0C z6}X|1J6+iG0Nm*=`Ct+G_*=i2<5I)E_DtCKfV+lV4)SQ%f}SYhZnB?{ZN-K& zt-GIa_YtZAXKW_K4Gj;eW6Ut9G0U6^3=zq}z7QAsJsbKDc1#}gy`5X?P!zypf_(&@ z0Q_PY!g|1E-LZuVc*PTXt$*w}&)8e+@cp?p-dnm08kPOqSm^f;kKG&|`)4na{qC`a zoK)UDw!C|6feYWCeQddR`gIj3nu5Q6F!V!i?B(d#Pah#E_|9pZ@1!%ry+h)#Injvt z$Ntdm{;^vZ#%|A$h`LS>P3EI$x8t4WRK2cYGi%WG`eFx2){K^5b%5yFvJ@mf|v*-B(;4!(g3HmNsO1U_@Cr;%Ju`W@5+uQlf zN};M$j_EEnVSU;L{Mvf)%I4OFLhLKoRaMO7D`GuugB}+^kL7$zUSkI^-6`cN@{-zc z=L}KXn98;0d+4h=^5V{93;Vb#XVS1gU+ZABG-(mNyJY3%;x=He9*7FHu`U$1t#Ufi z!8WU=Y!4b}YulQgXK2|Kim0wQ@1{ zcJSI4r*ai(DiN3JIhXJ8m;;|?5__fT+<{zL-g9H?|5ZQm=cMTRqDOpD8@ks>_d7Q- zvvDzVJ+ZPtYQ|yAw{S_;%2Ta4xtngFk9H5-{*O7})|KgH6>{SM;)~|8n1MgQ*3(Vg z%E1OMrkhedG2M#!vMOmgL!6qSdA?exI;#sajjRUww6J{W+xm7>nxeS`d7N#o?4_LC z!g7M=cc6$S-F2(*Ovd~wso+iWv`p%6(jt%RqWxltQroSA=NHIhCV#L4`7D(3R+)^mi@+o?*Os5m(@n?a zL*Hr3cViv)nnd1(J@}e~*;JY|8q%D|fPA$+6py5+<)+IeFe6tWqBh7E%qLxWbklKr zw@TY8^bq0;$_nOzFSTcbPj2uFU=7bpJTxo{R!8VYmM%JOKW%j>l`NzE5B9?nDw=Yj zoA}F@neZcqU|w^uJuVr>IBn?2ZP=*Q%Evcz{QBn7T46|q_i4xeF}L8Y>>>>>> dd45418c (code changes for aap_38439 from ols) from pandas import DataFrame, concat, read_csv, read_parquet from tqdm import tqdm diff --git a/scripts/evaluation/utils/response.py b/scripts/evaluation/utils/response.py index 63c0612b..f782a600 100644 --- a/scripts/evaluation/utils/response.py +++ b/scripts/evaluation/utils/response.py @@ -34,7 +34,6 @@ def get_model_response(query, provider, model, mode, api_client=None): provider_config = config.config.llm_providers.providers[provider] model_config = provider_config.models[model] llm = VANILLA_MODEL[provider_config.type](model, provider_config).load() - if mode == "ols_param": max_resp_tokens = model_config.parameters.max_tokens_for_response override_params = { @@ -50,6 +49,5 @@ def get_model_response(query, provider, model, mode, api_client=None): prompt, prompt_input = GeneratePrompt( query, rag_chunks, [], BASIC_PROMPT ).generate_prompt(model) - llm_chain = LLMChain(llm=llm, prompt=prompt, verbose=True) return llm_chain(inputs=prompt_input)["text"].strip() From ee64430b6e3653b869e781a0aae96da6abca7e24 Mon Sep 17 00:00:00 2001 From: Sumit Jaiswal Date: Tue, 21 Jan 2025 23:51:52 +0530 Subject: [PATCH 2/5] fix CI --- scripts/evaluation/eval_data/aap.parquet | Bin 0 -> 38961 bytes scripts/evaluation/response_evaluation.py | 9 --------- .../evaluation/utils/similarity_score_llm.py | 3 +++ 3 files changed, 3 insertions(+), 9 deletions(-) create mode 100644 scripts/evaluation/eval_data/aap.parquet diff --git a/scripts/evaluation/eval_data/aap.parquet b/scripts/evaluation/eval_data/aap.parquet new file mode 100644 index 0000000000000000000000000000000000000000..5be6eda5843bd8e6e36ede35b34b9496d3513b6b GIT binary patch literal 38961 zcmcG$d3aOR_CLH&djcoZYO5hT$ zEKs!w3MkGXRf`~^qH-OtS``(qI9^2shk8NfDmdd@-xa-I-}k-G_xU~VKQGT&5Yi0$ z?7j9{pY>Uv-C9?Do|!d2tua5Qnr}Y9m}U?O475zNEVOL231~TJxoA$biD;A1^3d|pCZkP3D?lqmn~F9Ktq83cZ93Wvv=X#Z zv@*1rXfCvJv{`7g(cEYiXmil!qE(_*q0K{^k5-LVgSG&z7Of7gUeC>FAbEgnHIQm zOY?C1H0zElJ>>@%J8#_9`s}%$(tSJHXJ_A3bKts;V;zsKy#GdD#i4ftkK7t~;q#w1 zJiV`TZjNxxi+6^O`-*RS?Lb6>pNMX7rqFz4@?UQ7z_@+s`23=FIU;()qAL*gHv1IO z6^#V_a>N@9h_yaB(i#l;%XH36qI1=j%}F&hM%oq8<85tKLQ0@T5u24rx1t2h3phF8 zG3#oUk)j2FqN}!2 zwS%-9J5!=zMf66>3Y=Sr$u9@wHYHR|(!)w5;tjNg%f_W{GHytp?`><3ggX_*qe?TM zo*RjV0wRvqBZtIq~wMt zh0Dg{NZmc1KG_>kRZelaYQ+UzN~p`Lbc>zLE2cA??T78rcq79}95 zy8;ZcCYKsNpMr1hZ4L(U*1ChCHL5Gj_hq)U%YimU3@hQVYo0;r>GX!&3scHU@XKZg zl~5>dir6l9DWWX41bsfG#l787DYnP~=2y0Cx_Rp+H<=Rf;Ec&poa;cu+oHaV%rlow z5_^MDd`qC!+ZGLVeaoada^|Hqs4VTRy{g#-&wYb(rLrPi0Xe}|<=FABO+wj{Lpd+-8 zo?}YRnHE&eo>jJBbU!igPnoCmw)lebnxPx~eXjSm=bLcJLSb?6N<*g-@_UsXo0hAH zWzVhYGA>G}O%8b1d0iyInL;dLM3MbvQu2vOm7*LDd)opy1)<>JKe(VMtF~;i6IGRY z1t*6Tm5U3P@y-=Q{}!_hAG(|ZF`*ZT(azm_s`AIMqAYP)uV&&x2D`i-B~)fNyICdd zJR9Hlu*ENTcA~ose3XBUN^FUS(VxBRl%Y_LFWBY{pht$#VN>fY=$#&KSRPqq@C93B zpPds$xuqo-4Mf8E%$U^s4DKGKCCY4M%aZat7)wX68SlV9oWUt&V*WUi5?kNRD3WR|Tt+f+;lhznv?Zc=Qqu{|FDDipeTUK# zQ59GB>2RkgSt(*L8tIHi#9%A4=lEf3gd4snMuMF|9C@#J?B3Y^@QAo2sf9CUm}GR% zaC>t0y1U2Z&x$j&2g8wY;nzgcaN&py_i~1|P%zqAXeG?orU?4D64F~qqVaRrF5=Y} zld$Rw)gY5Zl)MKgiAtc$8wv*eIQ(!?ACs=bS@H~D!OFhW9fipxMU3=zD&eveV~E3G zA&t+GTO!^rZ=_emyOIM=kr27)S61$H{S)7l{J=DPl`IB=5pSy(*C*Myh>3dVGIwR& zKr*9a%ynE?Y(M8pS9VwW89y8eL+U6`ChDy50%XA+sIYPx-7vI*zN-x-06oXICg z11;^8s_g}^R^#w$#96^W1jk!erN5o~__CYSKX)wAnC8enxu>`P9qSH`$jSAF_GojN zNnfM-Dm@_Mbal!tO1SuX#@5V{@J-)KQr~l|903c*txAP02dis5UnxZ!LHU$8B^zM@UGuf`kj-CGeI8L&q8= z5LPXaH`;$lT_AQwn|k8$x@4ah zDm8@|oMt$sIE+2G-#mpU)VSXZj;(r4{??j-`K$?T=_G;$A7kko*5< zLfv$VGmkzN|5r*KKFmHcgem_2kV|8!Ru-dy*tYOsh&M~RaL^Z3Wg8-~M66FH3X}U$ zY(prxw}P!=Wn(PLq0?eM$-w~A;=?r#wEY?S!kDJ)=+2~+ULIPqbOaUtBtD*gNx+-g3r982qe=CH7snYuwIM_SsM zD{{iUVGMuaET>o*VGeQ_cU8}jPV;a3a1(}Faa)Pq?Fb6~%u*Qr|BTln=1bh-K8(N~ zDNYwlX!?7T-_l`vYmu13cSt&pL5T%g%&$dx(Z)YQn-`;b1h>Qdk|w1%p2C9@hiu%}UIJJQ#lK))GZI$OX~m z_q*4n=ZN8GOFL#T@%dfxt~c_pk5?MmXxLt93VU(>6ji#2TO$zh;gZRx4WXcn(|4`+ zI}W3veA#NuKo}PhHDUJ1!720NXZaO55LUl3(;Gq1gzk=E6T{8gs8?=b1HI~{lfsxw z#c;4S(v8`XSvnU7=f!j)BYg10un+DteOo5}?GG!yE=+`7G)`(AHOK#XU5*dpRCsNT zQ%$>%C8RO<7ci?l>YBLLQ1$c}l5_MS4sTidP86G?USA~F8!*ltR{Ou_oZ{TUPu$Z} z5c;7HbcS8-SbC5L?>0_0DaD%k!8T?cMkiI{;ElR&rCEG;5{V8z#ax;a5@nCy8}J^# zm+exRYT_BrN|$|Mr)Czq<(j%1SzCktJww=rYjAc2@m4-y@+qy6+;-Vx2*_RDwqrl! z;N%3NPEl-Z_l8wE9l}Fp=-JFxOw%6RMwt86dw)7f+{dCLBdL`fdNY$)dNm=l)EAAr zO*XufNJO4Rg3NPSMlPUq7noC=ng#`7S40)j#gZDtacoH0elyMBMrP`|K5R`9}< zOvTye%n|>S^+zrx3gmDp|4HwndM5gT5@~T}IW_K)Yx^Wt_7Bu&Ws9;;^eC|~nKf-B zNr#9nWX4>szS)AS6~g6JJ#E2(N4$QeVj?k!&6qz`uej6B?MlJWs{|0l_IWS_T*0KI zAjECKFwvr_C*l4xOvBl)rbKbYd&Nw@yarbyIy zhP=!z=3y(rLW+6DzRH4I6TKtEe3Vt4e~w7Y4Cq14^56v$9Gqvq-MyE?tdkeY^vGUc zFM6e-tYNqmKjznVUvC!kp}EXlBIU8+iRx1vPK^D*K%ggtVvh`VocryoPZ z62YO?;R*-!E)zcO@pxUWlW4wF)-qcR_@3}YMs2_w4(F(^Az{~SU9TdC zDhbJfF&b70(AgRjI(B=ooB7S~M~t^#r*`TYX%ZW^kLz)Eu8dsmwNjRJqrWqrs z5>`@gtTN^R8 zU}%^B12r(bcj0o%@UG5+7`+QC7p8>G2wM$m6M_-Mp;_)sS)Ai;6z4WJ);7>44k2{^ zo#I5f(~BQETUim6MjQ>Ceklp7|IYg5zNthhc~XAS@}H`TnslZ}C|1L`US&P9y~@<~ z2lyI#ktj>F!h>plx-JC+RF_v4?mK)ZeUx=bX_Di(HKAzrjT z+k-*ox7buPxk_V&?4O1B@PpveMkj!zV|_mPu=Q176_321gpQxiA7J~(STLn4A*>4i zrX9|O&05lG|Aa&t?~)nrK19|Sry}mm;W{}Dm)U-U)WCKt%sSYJpBAK#b2`7`6!U*0 zMaS|7$h1G|V~>Irg%7bzv$2JZcB&M2sRe-qk^+%kV-uXgSSl{>WR~`mT(FlGY8YF! zG51Ke#z^EkGb=|6)=Iw&Dw8^~5X4M~)35ks>595%B(wb+GiAEom0Guwdab7VdxvB+ z{84Pce4`1G3l@>8?Wx5n|AS$fD>{O;q@236yb*wSg!{j16GU_FC`{#IE?Y|_|c9smBIM} zhLutawfu*z7?Ei)x45#rTI`l1;b5Q#s}~QlEm#MKBd||PzXKhj1_fFY4ag0+DqsIM ze_U&5SA3XmAF@48b$H)levI0p;Kud@JK?FsSSImfUDt0l-mAmx9if^9&>an~a$lh<(USL=m6BvmER1JYJ!{jziaAj+jCU!aqOk*2)DW+_xZq zH=(tOwwE==O{96`1JViQYS!E0uJ6cVh*!GTgzxeNqi_Ys&*A9HI-=Mgbv?pGlr|~D zfPp*MY8=ch&|&NhntN9Kx|3Gjm{_vk4jucM^cYElu7j}fZ1QkE7j5o<&HBNnOx)?n zmY|z@!_aTyLcy^5<-D;R4$S+rRWS}Q2E;~lWVnt?$r1l#HjY;s3kS~&dY%n<_}kR| zx8|M{oni%|B}ER?QN0cUP0qTN%&;VteeD=JtW$ct;oxWd_lo#TZpn({y8AZYm<%qH!B)^jZ)^Lz?s+AgGqvW9=NSF6i6 ztz#VXwDfZwf3Mm(%;cIan5Ns>lyK#p1_XW!8@!#SNIUN2w!&w%?81i1Q+?}jX};I! zod0A9{9w?<*evN^7^XsN>_0PAuw9dPbJeOiT@FHBTCC$obo6yY`4a}A+Ei6W=vs|5 z$d9VC|7l=bLe0kvbT!hGj@8`g%Z9?24ILS3>(d6wq;q?`5#fFVhro-*TT~X#HY;db z#vwk!GQ>9&Fxh6~7ut+24wsHz)Vt=BkaL2Xjs<2_p%zGI-<>c`&0#eGmsv=!Eo$NX zvWMQ_{B(ka7bNY=a@tt7S!v8;)WQ; z-|-eR)K2@7<&O-AGL!JD6QW8JK0XVA4{bY)U6N`~U~n z>X@%c*q)Splz+hJ&x9hELq&?O9*O@DIEJ(FODw4j!kj920VLQGY=AQaidh{;|dJ+mm5eM z{Z>Z;{Ihx}RP)zU{#Ttfpku=kiL${~dQ*aw3=dM)8##72T${{Aygn#Rffi6LQzg?^ zbWARNA;pCFNn9I+44yt_QW^|C1WWdB-rsmBtci*0rdU*8RF`v9UCVw)+U!R1?y6F$H z!d?c~CmFC1^|rz}4W<;~N*iJq|@bXYHHrp@|;mBYPP8^omKbDkbM_!QR}^}*Z} z1izK)dYs>b+oZdTONhy-abaYB>pPvhX}-%j8c!-{u9?`FMRm!l3c)UN&aYUq>h|e+oubR!0B>t?rIsLL ziaP-+1?WqHd(<2T)sahOtvrRocz6yhn2y8YU<*8D5iS7nLMWNYH(fvo<8hUkx8vAUz04a$8=X@bVpBcD z=GcTWR;fAN8}J~N=fUD)@FCV4v?f`N?+CY;vNkgv%t?|7XC%@{&nBtrRaM5%_>-EQ zVb&j{?S{O?Oc;j|MfU}j_278A9_-)igTkj4sMQU1JKnQkR_g491hvZxZ(HC48)JYP zHJrqjj*@IaM_z|T(;fHd=da|QMcdT9#J z$|0tfsgf>pU&h>Nz_K*h4eur+gN5)s%pXkRFbmp_B_Deat{#7!<9xxOkdsP+^bD_B z0tfW6T|Ed=f^Ay@%z+VHe)?Ei(>&tj?@P`euTg*LUaMP1O<_LQ1(3L`qj?W4*}Xg0gmoxxSk*>X840KMY(L}`u@W1(3U{~r zD0BP4OUSSb1#-bqW{2cDXFAyf{R6}f7;a{_0Uh4D;OAG>MtsAPkM?Y#A zd(kr1`kH}=!gIROl(Cv?=^o8gv>B ze{-5^3a^9a#l&ezv+7e|EhsPRR~Nlxaz>GyTGFTKc$s0vb_~%tUq1LcRmN(f6<*Vk zT>6B#^A0**1N!8{XRLdYADLv*sxc4UO!CoSB^h8mnkGN-(VoQ{Apq2a~*~QPqEWTYg6=lxnOr7e;a|TQ#iS0 zjkE(eE%899#Cn^-nL%3UVXi8k=^M-Z%wcmQ+F|&kl^~2u;GAdT9m~0@nej}?$ghS@ zA7@9X(tc9W))y4WJ{FX%KBX~HSX-u8UAH)fV`*o0JJ40}pnZJpL*9Cg1NJ7#x)|YI~iDRWg<`D6e-$m%2 z1maaqVI#d#UiNPW`h^MJ&2%^oa74r5-KY;6(1#HRuCS4ydQNJlEb+JNI59IUJmX;H zK$dHy4Z`Dw)1^v-Sm~+FvM=JW6z6xOpwFr6v;Rp|7wIKd;yf{sd_63+I+rmz5$jVp zkS1>w-ZS9j9D51eY5BYLaBT(&|HRr8Nax6XLti;xkFS27Y0p0v$7w19aJiazneX2H zq~_H)Q#jh$iA{%zpu>7I_gru43sQkR1=>yYV+%Yi<|A0arE>A+LI*2 z(~=bPHOwJ8Wa6C8BHrQXjdc7gPP!W*DYM*NRQX%qnbC4{MY(x@5Tq4*Hk*)GXb*Zu z>&?aWX5#A4W5sH?Nkd;(6R~2G)%l!WNOF`+)gf1EQyPTR>BQFeII;0swd)z{4&A#e zQ%ksDC~REGj3a#rO*k5-&XZ2~E zB`nEmK0?TX{Bic7kC`uQDfXf&44{y}+oR5OyY-y}geG6>iLx{;34M3Jz5hGdXQ+Cf zv3+ReH08ap7b7O7TX)J$1vvNp;@JwiL(d^0O>dNF6^B^>22%4zvhKIPsq*H#3ApvF zPoZszBbo3|y)Z+(P{M968CFJ1*uSoCN?6H-|7$H_nU#M%`_}_}$3O`V0iRe{!m7M` z@h%o#SjW+I9KvjVZQ5|s=%w6W*V~tJwA93JNhY?e8tYXCp;R;S$>>h5cqby2(S_`P zT^AOz0}Jb4WJVXW3m3EcHG7w`WWj}{Z2wYIqidFHR>SB`++Wv)o4CFr{M@yn>s>eRFfh9HgWPgLz&# z|M~JiUJHHGusF0=;}15qv(f{jtJ(j$F05t;R>xNVkM+UTuVu;N3$<*2?O!eB(MI;K z>p~;j-$>TvR*u%Pn`+s-2mboWUk~sl1C6o!{^JwzzL&FPPJTlH{4$NN%ycfv?E0+` zUmR#;=~v^kmu6YFqzkKasB?znHxqoGg}`|9%lW%rcfeJXnZ5LJo?f4)N+2P7IULW@ z#k#=xM;rOJbY!aNQ|b10WEB;-DhvPXI8w@33LSR1)nWLfk4;sT{2g}2&!O+v`UwR_v(Urns&gP+S-G02~sX{=;jwMMFI$NsD& zK~D+Bt%Et%=b9|BJ~^_{^r#a1%M@!DMd zUDl?_%ze&jam<^^!J?~#_)O+04#zBfRNy>LX!9ia$pPyo3jZi{Swg27IGaqGVO}{j zJ3%gh_Y1L;sxQ|C1HN8Y7jjhoW_V`;-GW?vBtJ?@OBsEwvKT&gTMtfyOBe8}g6!-G z@M{J}_4Vn4Z5m$q20r6V@KHG72 zF?Cj}TVTg{;k$YHi-~>*+2y5PO$ff%=2*3<v&SARQvT(rwbMbpJ#+F$2>^rHc&)9<4Dy%6lf*KAaI_ zK_8l7yRM#DMXHy4kULG+iljMgaOMO>YRFp9Ta zwFVoilA0ZQ8;2STepeIMjNP>qwl_#kHJCMp5(JfENj5YX@MYm*=d*b>%RG|fIUe3o&pjZ;r{+8Cv9Nig*{ zgU#jfh{oS^LDWvcfyvBC3@OQzh*)`>?eWG!KY{TJSvmafMF=mV(eNR@ka0^6`1h-|Dl< z$oH_gkpzvWoe6B8b2Nd)YbiebVe|{_8((O5C)iUJOO-9Iq@s8%OWm5hyR47t>V6&l zDgLi( z&gax?N39>t>M(Q2BioXS4`-}bW#uD)rwM$z0Y1-zq@qsDB;8o( zU_Fmv?D0bQbh$w+S6XEFd$F)n1MM?mc|E^<26IXN)N=I}l5_AXR&0hd3m{Ho8#9CM zov!{xd099=S$@-iy&Du~>^C(lDAV#no({Ll?uxog`JfBZ^hhhD7ml3Ng^@pi-bVdv z@USk{lK70Dr=u^_w0=YwCx81q*y!fptR8w#uq6zhwNJN(%Hd%>k_dj>nn2H0C@%O_RKysi+uZB&Fx>gL^eZ;d6>`bAPoGSY1u0o?fegZF8}7 zK7k?TojUl^KwszJCN12cRh=@Rq+o57UaiT#fm`@ICmiNfvr}_z?{lE8Brf=1D#Ynx zE|6<`p-5<$4fFK$RyWm_!3(R@TlkD-$5nb7SPZ9dOJiSY681ar{T9_FmgKn5bH!qY5RcGj&%7RDIe0YzTMB_0 zaf9oz;^FlYe5HdaM*qifg|6nCV(Y~s8c3iO67#S%Z^q7W$Nh0IW2Wu=Gs)g#CQkNgRo?5iDyg24o!DhPHJ#1lHe6X~M6Dwz{ z9?^pVd)6U^9w>otSH@^ae!)WCJlFa%&AyZucyns>V40K``jUapnRHzPSqn8QU|tow zMzQu!MigPHv6)?}(@vRqZ`jmAz&@+=`8Zlasar?8=ByjbxE@iB*Kecm79YP2*Y-csiGm27zjIfO> zS7j#;(0|QjGWL!gyW$@B=S<^Ap}PpP+L~g*7Zg=F;=B&+TsXXv-|ImlH;gP_SWl%) zel~|wifUH+&7$nI5uF==gTSM5C5-S-HrERFGUw5qAFoRDI8)Q!k* zglAk*7xQXhTfFUNH}U2lowf63X=mSF)tlDnavk=ZUACj8j?4;mU%*fFIl@B;MeiQJ z*LuB$zkgQATHI)n_T(Z>;2=_%IIPMm%9oZMaNci$or2kHOO5yJ;Q%&W!X!uiV`d+oP z!1*KLZ&$NePfp?Q<6I@hO1yNe?mBk~Qc8MxN=ZT#43XE&9wFiada+um?Z3!Y{ z*G6y0UT+Uvu^QeqLmgM}#?Vg`ISo`Dg@dY8xOW-sUdD$Eq{r2a38?o8Tl)fYMb=8I zc|L~&gAZ|wYFTd^btiMhV!8E-R_24XGlem_wRhVq=wWkcaU4IWBjKekhJI-D3;Y+e z`4I96`ByP)rounkn*FVd^xF49Zw2fyyJWaon>9mviDXIJ1~F{TrW@xR-ZHc!|9h6s z(4jcZ*u+L7^p-`isT?P<^aj$jL+UAMFy5(Jsp{%WboLOaOtN+>{o7d)m%W(u@{_qG zi{U(dxZp3*}{*MeNf9LR4>V*G`rR2EAN_`WGS9bNi@< zh~>MgIj2p-zd7G)VMQAqzbr2e$-hUcZD%>#w;E{Bu&7n}EjjyeWvP?UmPOw`h8%{u z5e~{0Y%Ns=T7$xxW&C}bzE^p-7QW{A&jF>*a~rGaO)i?U+Wt7OTI=0O7LPX^6CN1< z?tGZ>?{kT;Xf@Sq2c7DCdi6M(-U^FFF4z_^9 z`uU0wk=|nNEcO!;u54R7LVwT<`V4ZQHzTZ?)bA&XvzgEXt+cJ$da4w$)Sf!*NmxUb z11qMWxEyt5b1EyaVP7)`mg)>*Rjqpt|L-c}Y`PnTrYN5l^Tq0N*Ow$L<7oicAd&#x zhsq6jxfbRulrMoRT41@xBv!T}kn|U+?xHKyc7vdw=jd$8ek;Lds)r}`ll6(EYrmi;wfq}t zD$!XM=U~BQ+xx`=EGn#LD{xJlTA(c%n|5vUNvpeFE^u6^pvZTl=;}WHT%GVqi}J2y z=1@EWJCcbH zA$93b-Nj21;0^7D18n)d3Fv*tdd`1a+(_uDd5GTLt}gv|6{*JgVzH`FY3&oNpCebu ze7;>**f4)MAgI-@Pe=q*EwCzr4P+D$fYt^_lCeme&6muzXA{3w(^H(w=;yG`X1Fy? z2#M(Vj(<|u-Pn#hU2S}SYo}R#LBg=f22y zCGwveRAqUz=o0B0C}_$(H9Q+i+UToTK1h6mXuCs`|I&D7XPzGRiq4nDWCuCB3n$*W z&i)47P$Ny}P?$uEJ@9S{jMqX|wMw77H>R0^m0HKkDaPc0FS>gXw( zynywHDur`Sh@&t2TnCRmPkQKxspHmo5KPXA;mT`BXW>LMj5YP|AR+5!1M{TqnBiHPYkc|4;5QFyJ9sfhKBwsja&`L#(g zMSW}3r^?E^HwAT`C`MKy{E-Cz3b^KykvPU0>v6ysuMUr2^sen{S|oi2ot&`9<}LxP zSM`QFS(upU*r!F%TCdOkv>gViy^iro>Y@f)`%)Urp!XX24U3t5P?TU_#5zBEWn!Fl zY8&%xfxU9tV3I34zdp{LraEAIp)U6)P8L1FRU#It@DFkDRR0}xjJWd|dTtqR>0cAs zKnoo6%TapLplHFNOTN~1xp2kmrUw1o7=f5f0PW2DVe?=O$o){6%lwsD)1wV$dKXy8gBN%_Qp_I5L@y@Q{UDgcS7d zcE(V!a^P+FL`JUkmxO9mH`sBVKU}lhw{ok>c6=T2WdFLM@E%k0Hr_83ezxfNIQT7r zLmg@9qj9M>t1IYsX0f0o z6O{yBSl7b_MWwNC0@yA?iOt#o2^l*EAESddwk0z1k3J`csBR7GoJ7^D*Uy1X3H&t@ zHdXw5~=ApLU-Jey>44clpaWx_acjOAGXE7T(3OF%o9HD z1D}TZR=!lYWD$+_Fc#udyN-X&bg4@6uUW(PI{8<%M0S3`z`K>yLr%oF|85r!(0gV=Z&UM`}-a!#r~=e|_)Jzty>pkFwP3 zaM+2<(78`%aVS!T!vPjmJZ3*>De2%r14(`&(^jsiL)s|aa3sL|Bs6S9kzedw50Vju zdx`5tJgFqYq0JmN?O?3OfTCyIPz~4wYIV|o8irZm9Ox$onN$@;7@SgHvu>@~-WwwKX5d!OJ<2nk0O(9F}SgJ`|UGibHYI?OiL((k^5K zAk9|(Az@1T_N?y~lvJq(^VZC6T*3O=3BS7oS&v(Ir@*c1l7+7%ozLs{!qf?>XO?C# zf?HpCn>QQvOk20uKwZN#Ls8h$1AC0j5b`S4q=;vz_*F#+I2hr)H zFURB6`f@xec;z9L@SgVIJvMJhYNcIf=1YWWm#6d#mi#6~dpf%)()%ak?)|hYOQ;9u2XnMrIXff1ua1N6zhsr(38f$rnBZ*>9qlj8lg^Bhv=>a9LcEG zcss}ZlYAwS1!o(oJ_K`=-;xG*Z5n)!Rs5+%BaaGo<8d!-Xh*T_z--q^I2VO6YDhNb z|Ez=O?aYIa5ja=KA~}7t-g$O1+>(*MW=yQTd0z$;>R@}vpo4Zv&Jn%i3O(y=rH2Yz zAKoVH;9!#Btlx1~&Ab8o?99U>O@=?tG|`X|H*KDFa1NcOi6zopCkF7XSM{Ij`v!-x z-we-Q33W~^sB3s>p|B~XZcX+D>QE=G9Dlh}Dne~;>F>#@^K1(<_AfP)D)9AjHc_;x za;19MISGE(kTuFwDFNQr+3t1J^+_U(Po`6Jj#=8{*1`L&j?GLK?9j=6Mn|Vp`aSOU zsI>BI0so`Z@u7yvl7dVDv%9jL9?SrD50hwn)rdsP8}3?HW%g2S<<7?mDBxBLH4Fm7)BC+JiQIKDw%EcRk8Pp0p+} zgK!!Qndtx*(+hXf;UYMmBOKX`@#^>#(#t5q%1nBt4s;nrG0sI_cGl_gdkL*F#?+qG zu!0IttR$jC6OxKS(F*<$>81@Eh0U$R8;;WJdHd$IKf)714#iM?&g~jKWv0XW!ap4I z`*Fgfx^tKD=O^?Z==(RnNM?#~aQ1ZTLx~6vd~kn)os<5JYK<`SYkq@QYDDS;+YptJ zJ_iXq&U5-O?AALPw}_~SRzkUvU@pUrtcRVg7(zA$P;e3HhH@(%-nQ^pP31P#HCz~3 zJJ_E5o$!vvczJr`{05X^w}g5-nU(!V;L}TyBNSdvMlskD1!FV}t7aHlWIPrI zb0R#WW}X}rc*|rC#gnjRDxIljb`7qQ+i8b}K9v|No%!X7DxlxQOJ)2XbN1HMFM>`%+G_@8(L7J>l{4A zq|=|(I99oX))cw-(ii%yfi0>oE7Gma-dui}JqF)KE~9T~xLo|U1L^V&g^@Ueh&sry zV`CA2e=0M~RiF&%z+%4hT1&*4f@eyqA-#u0c22z9!CJz-f!OgJI9zH!zg}PlMn(4j z`;{-Utd4!@q5(E04F4SrdU)W5?Z>I-8tczV!;w8jhxa3Z3muyC`=N8xMOqET{{+&2Yj7_Df+$ zBCRxH83gB?tT>n6&@-6B?B)+`hL9%vTi#k0q#I0ycXG_H!>1i{y7?*&O03vx1N}I@ z-NNhicuIW|6X4}cEK^mJd2JW#4OtIOW}5i>v?P>WY7sKanN-zT*gT#^)zTUrP1D20 zB)Tbuh{yH`bsD~nx0YM7Z2TdkYG&~hylT4j)&d%|V3@q}di9dryR2uCW5~ZJ!#EY4 z>-}-o;|Z#|+#lhNWDZYO3u&kxY#C{w)u4LAnz*LWHqKrG56>7G%jDTtLOM5kBI)`Q zNw(|VOJ7a`QIFa};ra0+qVS+rcw6m$n(q^=>&?<)R~7$2HrROUkcBAcK7~c)l8FtA zaQiIyZ4G}~U%!xnF!vx3FNOu;%hra>2ksuH;IS~zqq z@;D{#0+!^m)+&6u4R6g*`ppk>VJ#gaQZ0 z2zOk!^ZERceef>+uAK!ls2b{$_?=qJbKcT{IAL>?8poj)&ac2(3+gmz_4PxEa-l`W z9fb`-{`Uw(uVcV^hR4`>!$i7Z0Osi7To!8HCh`T((r2^`R$Th!tn(AN!=ZX`cqs{0 zUhSoQGX`_brL1lXRA38?ihizCfUwt`sBxYAlED-5qDX;)WHZoVK z`sk)x=A9oE zUfW|m1*vQ}E31-n#e3IkAR~jmZ!vx=G^hoOAAQJlOVNuMg~P?FFccd3Hw|!%qN4CA zY$jq;zrD9PWLi}Gq`>dgMPU5iK#9jO|8f9PkXo_=7p zzMz49X{3YNvRxyv-2~k^>c7WMCL&{pzpwxzHac2_9(-&I|A4u~z(qENy>Zst)KJNDczjD}(jvvK;~qNzJN-*m=kqElLE4@F13Q#? zEFCvSvNdZf8}b0%>MSLyH>xUbq1S8eL!|fIYt}Ec15d!V@z$5KVe7=K2jFB5HX(i9 z<`9%kgK5dMy&7_EjvZke;8fRdpXz?LbVcs#S08*txH?rHQ}wMx7wGvP_!tRyUFk`w zbeG<7R;ZZ8K>w}q&&mB2u8qP^pcBiZUf7yR8*;4+5(>|91w2ZP;md^A`yhCuaOu9j zVuwG$&cL%9RlUM4Sb3HK)lBNhunukDlk}Jhs7A~>1ef1#?vProJIvrW@cj}^*U6gl z7K32!6Y(^0Z7)`f0q2Vhykvs}gQGCp_KaQsCYI6G?p$NniBt7|iz80ktr_)cgr*zl zfsCxwiUS#gQ=nLYf#k~PnX`MYay`RrrbG2rS4#ZDT;;A(WFcl8&s_2eRAlp-NxdHn z=K!pY!pVcMI12^t#s2^fPy7IvMrYh?EP=_KghWY@7Fy`M+3U0ZzLxbnoi}fE5%0-i zGksRWH#kYe`9@smWW2L)Y~h%A4o?^IXOnE2!mJ;rG~1O-T2<qcIK%qS)T` zEsbk5W!9IwW@TBM9LPFD#eAq9-+}t%!0@-cf0HPNaX&W)dzAf^V^d$jW}A8re3wG; z*W@^z+z0D**{39T-RpI@qib{RweE*dtg*1MgoK5rTf1_Th`J=l$I9XQ4bEU`nTI!E5G-n=|)8ME1%EF-|*_W7dzZ4!X?b`zv zb&jKD{DER*j+W6o06D?lJbwfXKthhvxmEgG5X`S}NOf@5gk%GriM}R|o>S9BTH65* z+aUZUz$<0=<)u9${sIR0$_V3iv0wfY48z@O1fQEH#VC_KM6|7UI=2$nop60jR$Mq9 z3Nmbub1+to6wn8Ui{3xd0YVL3SVOz?<#FuDH@2Pv`uF8N#UOx2G&jlyl3k;=)DeDP{h_&9^6U4j@?3GrtQp?Kr>-&3hX&R zbN0z&vTsd$L-Q=Yk(LI;LLmh4(>ZTueeb4U}RxF2KY4+K^Uy}kWw<0vWyMlQh z{ST&Retl6GtEakjzSc(k^phjF2QPa0*YkBwJdVzMh9Kual4?fXD1W6n^)1BH)06P< zD^1RZ&x~;8we&Ze@Z6nX9A|CdT-9>BaewwkhwYv$cl`Dym}Z7Gw_|c}8Q`Tt*Xz(X z-Z`>oAkRNvDmB`j_q$0>!PEKq@ytiqA+=sK1Fm117W}IUuNrbKgX7kr>qcG|KB3I3 zu4>0;+3+T$rgHczkl=~zss@sJmYq((9(o9U7!Vu z)0Grjprz23rA(W&4NVu?rlkliG z$y^QO^Q`V+_WsDexOq78>%5gM1!L6b^F1F#yJM~)Kib*tL`}G7Qkj-m|6hf{S|qpD zHIuWEbU+TQPF$Ofd2iQ%0b~clIb!I*wU}Dxo*-3YJa-!_qQtvbsJPTw?EBnyqiK;F z!rSXv#6J@eGZjTLXcS{c(w5QsS!ADq3>kqyAFegZIDcIr8M1=kI`7OOs=N*J{0;-x zs9o*OUd1$99$Cv4$~VM5KbAbJu)aWRjjTvhqqs+l3`nqOazkc+bC8t6P*0{?ggo-W z8cA}hN>U*AqjNT#()8PKvO82(S!q2@mWF4C86!}nHmiGwLOhc|BKMa?`XC8=S zC#-lQ`#c*)WLjpreXd2)$QizDY}g)w>??M&MGP3m!? z9L3&UhhV7~jVk|HTECV%8cM8MI$p(HpDhd}M;;-W`s^oNcRIdvM7cfnUW7V6OJ|s5 z^$qlkhv=SYvX>$`m7{6od>E4E7AIH@DPBGKWj$dwE7Q5>EUU(I@0h#pg?YKwTqTkx zwvhFqT7PAw$3rZ7;IuVyV>Pj(G-||wxYG(5qTfopYO-*Oz?2NsjVW-(G-df)+Ua-` zw+abr9&)vj-(vJCVX^u(nuKy#EGG@h)1D9bS-O)2k&2Y60U0}{M-_y=owNJZ8r&(GFa44x)OVn2q7eRB<2G${LLMzkDNby;Tb z8Mi%!4HXkil^RMED@K{r3hO}KW3dBG>i%KOpaB z$b4?@;AA>$uyNE?ra8*Zzi_||jE}8X!tyo?u^rrBrpW-KaKDz!Q+3S~e~6WPa@O#JpCl{ zbB=L$qCS_O?$X}#4a2<7eG)P67a0?G?4^K#9(xWp({}cpJr`NBPm?PO0GFpe1@Abjrt92<;HR17m)!d@3ChnJ>5VbQJTIfB8vxV=Go?} zu-1*fTylJz@vaTF3N2YU3cJw~JuE)4gBt0B3bJmU?f58pIM!807+e+X+sP|&PweCd$&bXd}^{ON6FtQ|B-b;e)X?3YBcOoI%cT; z+s<}w-eBeAjzjuRk{g}8lC;CO$=`|?ni|C3w~YI2&tdqZX{s$U!Sq@wnIy`v!8}W3 z)goD`+Mff{8C^RfeSQdJw;x{A<>=NaJP3!p$}N%rr~1*>Y!B?7Ouo zFEV74x!Y_yHducq{m~lC+_0^y>(v(X@J4c$m-%9Z&*@uTtc$L2k*Wqqtk2oV=xaH^ zS~@~=$W>e6rXWi<^?Cx|Y%VLXnNPWrPB9y_M3Ri{*<@9$EG6-YrJXaSknDs${EJE zxR%UJWonkcaZe0^t#1QK)Yu*}kjxko7t=Y=B5D`5gwn0UW5Tygq?477)hb?P`gpuG zhde%*JXK6LX}PZwY#C0v%}#Pc$xC76At*4$!pQnh2QX&G+($d#@4k!dQ?iGmXUQYT zU3pNOlQuJr^zlyU8~0Ul<<$Aj0L0ZkPPClqO$&L2mSB^B10;phdD*zhd7laEQh_UjT5KqvZI%p zwyC-3*~Ue@@rF?N?8PW@EP}`jwA&E?oJsDnQwj}$8?b}Kcd<|CW*NWQNM2B~i^yGR zBsPe}T2|xda+O@y zQ0{&u`C%kn`0bITNlHt1)`f6ShGD<#`a0joEs12~Na2v~PjnAEYWhWQ;m5VVJ?>Ga zp?ycnUm0@pF;e)5>vUuy#%`nA0fqLZ7*QGXAjGzzk>t4+C_fA4%X}gYRUj9oG4L2Y zpCMl(Xvp<$`n+Z9xirFhzmYcxhR7A-=PN7F{qF8l>S_l97e*8nPQ2 zid%lI%F*>wBtm9s9=S*s2`#e+CBsxuXvq z%H$u?u;PP`a?z7Yx(|6@~*!dAn{svFutt~d2TuK(XB_fkBa z%o|U>nDWEq*wP6|B)o83H&JL7_6b>B9)a(J)}}e`hV{t#s9dw3?1+(7*I+9F?4w2V z9b<9Tu;t`d#r$bG3tlU^6*Nnl&a_({bqc|@UGN^VP~%ozK0 z`TFRW$akTUcgSxgMq=Do&L0XKsz1z`j)cn6%?=lXV6U_Zkk+n$x_uvlFsFWy+K=oX zPSh`B5skAs$M&=AdW0YsJ67oG5V$xyYIMh%BzgurEvC|+i@I_CMvnue|ySV0_sRNwX(>uh#rGC%~#=py>saQa|4 z5BWoaL?NyuB&o>L$xVak4Gz*en7$hUJSA}r|DHXTttBr;LpA;39eJbfY0dG~_2ltU zmO19YKbX2LquC5Tk8Y0@UKXGNGu3j!G@NR69@!1Ez;1`kF`V0?XVc~N%z}*(^%oU% zHX)n*#_#h@{BAH14vL{UjUl3%oM|k5l8NhR4T6?+ zZz9blmR&rTI)YA6vIUO$q1@vhT18UzaK%)HS;!IkTo{|`I69dAsA6K^Sv1|(K^{3` zyY4yKe2nho$UD{KbaPcM!_|%;4{6AQ3VOYg{=J&;lp8&e-+;}AL@2Z#dkf&c7Mm@* z2-zJ*zKx`LhK>C#*K)+F*#GGr6O0GrxS9lnRqU!NV$65M0dIkvZbO%vQ-z$4Un>%1 zT=9@Vd~BP7+ixSU*GJ|E*8?bFQi<{~29=aKv19fhiG%csr|4T!zCrBW*~WQM#=V0J zV+Ig=1l&H1&y<|m*~E_HPj-sML#(+uts%sjkp3g1c~bu~9ial|!Z|ysGXy zO*d$a4WsE}W>WezxpkWS4nnAL?LK2pD(o(%^zK=X42F%V%cO5;=;Pr`yOZx{yGlVK z3;6rH5|CG4o3jUzum#31BdjiN?jgyi_$%SWVX`UgTL%_e*AjEa)T89kB*!aZ#)YA9 zJiI@WF5i2Q%`LL8BT-5~Hqp&BQ1A}UsZR3cN7exF<~j0T8Blw?q}Q2VIII|DQJh1PD2us#hOXRgweT+ zowuoAp7H{vMYjklkM{}g0VR1`LBi*mt|RteGk&e-tDIL`pf-_MrC{5Lh_{6?$&AHx zL!_wa+$BYRzFDmK!SOR1%|3@Qn?IZ$(aP7#@*C*hqb9GiyE0)rV>#JkymJV>H(ow& z|3tolwqf&)q+NrU_KrDRS_JDH7t4l0=lCenZl@n}>?*o`83CY4_F@a7iBpoXH zsfXlMZXDn}zxwY?4dsR~?OURF4s&yG47N{FtY@ztZE2|iGFmfY#w=!UWdoV95|OzJ zj3hlh?F1wCCpB{rZ{lrAZX;<5Zv1dzHhHm_`y{n#P(Ly=oV043rhep+DA#SVPw8S3 z*@4EPCt^QVAo&MD!mY7@LGv~d8jWzT>nPL89b9G^8$m1sy-xxLNqB<}J4An6i9zGO z(pkT>m`CkNb~N|5PUX*ezJl9@kK)#U()E<{l+H3%%_pR5JIvey7_Z+c=s`{wG9j4p=Mo}%Lwrdy&l<@A)2A0LpDS4&ua z<2guKM|U!n+@XZzTw;(LW` zy3X~9rERh;G|u@3bInqQpK7|Pi##)a^?v0;3dhsT#f^%E8x@vs5-nBXOf`4ZNvTi zV46GMF8vFqr<5}=RkZH7^;U9QDE~{x_2MM<<>092HRk+boKt+xFm z_;llD1)JosGJN@|=Ex@Qs73CNsg^;vpA+qzs0gfMhos-g!UieMLc-E}6G?IOxd^gT zbLxy`q;g|Ss!Dl-({t8(3!7@?jYo4{Pu_TjWI4!^@#MB4q+cn`{)8OJq5BdY;Y^Ip z$Sd*|GLF5-<{@*cWS$n=`|-%fS~nA59^F5Yk3Kf$<3zB^#455WW>1LczRmm69%a$O z7aZv=j4-$BQ8+LTJ4i*iag>KT5>jSI9!u=p-{nY|9a7cK*v?AB>1rXN5f}dYgUO=~$C3xuym3$MoLd0?>)tkRSLQ+k=765nx|QB zv!ukYj;AY<+pEbijSV<~&0;=;jQM9+WYQ0_Jz7h@)U2wIRrpEZCF6aerWDnx4o+Uo8 z_#5K~*d8+--f9fQ25uzg&_Fq>-mKWzkaD*u-wpI9l5>k`9s)a?xI-VB9x%zo3Q{-A zNYwO+N?N9%OG9Y`P*2jeaz0X5#okViuO{mSThe|eCwUlY3pcGg&i$g2ClVP@VcOH6 zR3C_O>=*)pX6-0aVaV3VD;;eNm-CVkLhrCqWq8+AzJJ$|*fSW3gCR`DM<;A&hM2Y~ zjPH&m4-Y+@#2%G-eKbFsuC@f$2Nu9KejD)9X_Lxf=SgC|w#8FfQ_uZ6*EE)Q+@)p& za&O&GG0rSB`77`by~oB(Ars#j9cRu&m}Ue1X@`efAcwzDXCWkw>>sTLx?d5AZe)tM zle-lKP$iMRu-vVcVEr7?haIc?sjC_){Ukqvo0i0deM)LtH*c_H>h!xw?q2S~P_j@( z-+aeZWt6oy8r=!>{a8-8&sY>~*=-Gi<#@?at7V5uxk>Go9y2$Beh^A$MRBi(0y@k` z3Ud0Nl%vFlav4 z&@YwTjB&KT(ouqB6fW6bb$|hsT|3>mitgidt92t&Z?oSdndGKt5SEM?MPuPiUr0yr zmt z)!vr4?5a%vHi|^VRA@uUY^CJ^%fbuc+*c|( zU(N5OL-d|)^hL#)A|`I5+gOAEFmio5jSi#hh-sIWep>l#mvAr1;%x6ymXhZqvcE`p zUs%#9%(u%N*KxPoIh)EkCPYlreVaF=ANuKb1BqAhr7O~fM=m}$Am_OO>sTg~8y+Xp zoa1IqG?PJZKVwO)X5)}8xH#OnHjeC{KthMgEu{n43o%+fPcGVw5#h?u0_Ti-E7fqj zw{V778lz45OH}s(az0hj2v1%@K(it5c?Jl!oi3Q}=vf`zYG9vQtlLMPcOE`U% zlEyh!E90WCs{-|O$vgay$aLirhrn7iCIS+2Qju|OI3Md+TLO1sE-k+*a=*es`cvmB zlgL>DYnZq|fuYXR1RUjTvSFOUp4CLlj^^LZ#Cfi^9J0sGtdIWLG$hpf)%Kn2xAgmQ z^tG?ay$0KWVsg3^@l`%ft!3}TnlFrG?A`Y;ewx~}d9OItRP<@rMFj$3=vPs;>sJJH zBvYpacDuVFa5ON{_MJh0){|puA1}X~3|G=Gcs`2&_eU|hgH`){z~*@EYU9fWWdF18 z$z0PM-wh%U=#4kLEjQ6lwejf)uCd{~E; zj3z78-mWUn&69M-&#cr7*-1l_e3z`yBMusj)Vh-u|o29nC}$j4`+Gsy^| zK(ZcLw;VF$yNQV2;BsdgrzID5gpg|l)6yFew%b6%fZb~<1~Q(8B%~PY7Q%{ES&-5c z!tK(-u3=3?C@i-ig?-Q@dS#XE6$AQlB_$|+vYvLNWEUB&T5^?@i%et>lWA$fYO**T z3eK9(S4C;fg+QL>Zl1)QO3of6&(qA5uaRHz+s;ieKbxj9vABvz9GsQg4>H-2wo=Qz&vQ?BFe>Z#9gXT$1F z!;@1b9A-uJ{*UQIrSW-vvkJfPVHX$c$ z>Lzb>HF1`$N)&h4o*lVZerz~>Um?Gmzo+B1V=0ykAJ92E+vyy3Gk3KbgTUTS*Xr53 z`3Yo~O}r8C8QmWh#OIg+zX4c%VEX^PPJ%1}xr;~QF+koTC<>4ZCz>VfZ&V3zrhvae zZX<$P%ss>e2zQ2g#NBKLOdY^QdKiqv3dDpVfeRS>66CrWnLhwosc!*nF1inBvJwXm z&=2MI2#G|;Td(nyNNhg%Awb%tHc8|*xJ#^LVm>W2U+#1iajYj7Lli(e| z_rhCtE8kr8Ix{Iq8iW>91Je?<1?^5v3(rXa<*1qbP!yU6#9$1GOR`5UXX@EeW}xTx z(82I0{a-+C6r?RegOZQfvju&5?dMTa=gGunDWTj z0;Ig=f>bl`41|D;LGn`oJe0j90^9=N3dvsyWc?-yS%&?@049n0W`7eq9(mSg2^Pp& z6JH0=Pjm+$4}k2S@f2efrr;?JU@?(cf%>45E0skPFrI#i`$FQ;aX1p-O%>!>*{?+f zDLW(_@Oq|l>Nm*Fk@LdJI26&x^AY(})D&M2Sr!XhGP?*MKTUEiw(|Te{%g^g-JrF* zo4l<;L1c!6MegxA;Jm*f<~m^-&LWUMm@BNUGVnx*2F$k>kI&b115*byUVjtcZ2)wR z6pfrc(Eq**xnCk!d0LcrMVK3;aDH)Q6l#Jdxf{G`Q=^l0%JmW;9weA?;|#qN0)o)N zT2)wg2@)*0a`LMCA+5Bxflw_M;DDZ%=z+?3et>l}j4~a!$aT>xg)iBm**gP%1mIQy z-@UrE?=iWT!UV`t~eZCYgd z#l*=|)o8I_EMl|ckoQv#5)s_d;!&FV7?h$1jz;&EBNiOX8WB@1;*ZB_;R z{SsnU5a%ce^e)U8ilkOBD@fca&N`X*M>d@i6A+GM6?tHI-3Kh1th`Z-%K*3Fx_%E%(#Ww5?FunD_#BBQhi0tUgYi^Kpu_T!*&dB!NwuM081dWTYk zWbSYz;~|Ct2{2xd@-F@=HBkFnrjKDe&cOTQcW>S&+_Nf?vFknqKy-z_zd^?0FdaM z$_`AzvX*#QB#;(B4|4{EXNzWY!3h}vrd-|M8Ni+);e-kM)pmbl9Dk1%ECZPKb0V5S z#?~V8>+h%r1}Yl>-WWTCeG)EKz?;>3I_5HBX-Gl7T>gko7>;e*1CZ4whk3?|?Tkn1 zLw<8z*H7u?pn{Os?b!y%cL{jEYlBu=?={_tAj-hhL-@Y+J$@3?&^Z85z!w2a8X#LT_$1GIGE2dZJYE9Lh!)Okw!9 z6<8GA!_^qczmX*6vu13jy_{6aa}hmcJ@g4x{j3mVKQLnVLkLRuN!If(6vi)=z^tzb zoa%a23n@XYq_-;P6=1qp&Z;&FfE1E2)P5GhP}EHzL%`Ram3l5vVxC4Z2RrZegme=_g04G0iLa$6<$H-NG8C z)-*>Gh{xR4WpE%6m>kUF>K+pxMC->M>DE^tkfYU^1T=#tuUEibO$3jw=;zPn@M&%^Cqz-w=$j z^A0r4RH_mD?Gcxrd58vl*;%#tSNO6PW^C zkW7Rhx*ndouXPXePqCHK?$;ccpoMVMCsELanh~uIDwk6!hmEt3KygR@BhqVL zb3GQ(!$i0OlCj?5olWK=Ktw|5m>TPqY|A7@Yv@Hb_LQ>^8V`i>+mOG=67r=&SO;)l z%u=%mUKVgQu!ZbfvKB9WQOmfM4cO9J8$h^38??}Y>bt(zLs2Jhgdw>dqo;ri1+ozk zx#2QXxRy8{XA}+1o=BJX#c_iaklU=U^6$p?P!1xDO+pN^tKcBrXpPS7pG@^Rz}hu_YWwEV!fSN{w}urb1X20l^;KpZzq@ zhQvecyM#nA?(N6u(-dK}I8^AP6wp{LM6HG^!|RNYPXfmVWT0lpNi+`JEkR~Hc@0y% z@oeCli-*}X=wGvyduX?Me-XA&u23;=kpiJk(d$!wW{ru;z*cFIf)R=wiCFcua}*N+ zd_?SJ;(2~^R4-^`Gno*Ea4+$Op%P6+kc}xWdz<`GwFr<#*14yN@pZo1%}5w8K9Nih z$C7S>S_G&?7QO)SYy?y!a9t|3noW*(LYh8jG1pU`U}8+kFSAd#rpa80WVbabqAoS% z&)NLWhm=apbA7VzWrDs-B5!cVzrwBIXe<2 zO8Xu9OgaIivY?n@0^(B>LNcbIu|0H_LGAg{h{Od4^a_fk-VOuV9mEg>7ecc_L;qM! zoFA8H1KYelUyowl(;=6~`|@Dm0TRH351g34|4~mjpt+YzygbO4nmnzc4Q;{9Ky4t7 z28jzX7E-+_=4JFOc1|MKVByL1$;syzPduo9$3Xb08%Ko$AkZufiP)~ZEG;q>1 zKqL_?9T1!(2@SI?$e0&YYJn9d&HUe9!jwTTNzx;jNdBbcpbCr%OKS`zNRFZA9`Yx2 z|9r@*J*2G|UnnX)s}fz-EW*(DyGi-Sv4Y%R^@lR0`4wz}2WAYE7_1+b8W>_w^}Z!4 zJzP@Xf+N>EyM8yhF0F+lLWMJrRu%z_=w!1*g zEYTz92MNIXb}?x=;F&$MAG>QEV$1ro6$S`UFx*M|5l$}f=3RM1 zaG<5m{}1{e4QoW5gO!x*Ba)fs5+oTmO?-k?3o{lKK%ZXf=-^NV>j`ZLX!#7~U1Dj2E2-ImJp(h&f#!^aIVyaSU$Wc<>oyW!mE*he-t|=; znBL-sKjzhKI~V4^U*L8@%_@8cqni3N&LYwGM9bVK1-@s8qat3m>zIW&2G4K!`a+emrFjOd)!i{xw#=lD#z9JyFun zU}%nD2TbbgJacxCvlf*@<|<=A!O}qr)f~KuJ+`IhG~hW2mz3H+n+y?`$Kvwi4E<2l zzpR~BA2CTE5?qLir~&w?^uqHwz4404W{@CRgRjf)IA-{tT*Z%6!+X1e-Nd2;A{Qky zhW^d4I@XDLH7-~~3?Z&8Mz6*j#Z-{oNqdf!nG}V4gBSbbc*VJT<)u(e=kg$}uII|j zO6hWox5d|YReCy7@+-?5AwV(QCN?rE(r06$P>*D$zcx}TS|5VzK#t{ZR@gp~+Z0g- z#k8K*F$M)B@Zd{o-XE_rk#QJOMjs;ekrw|jjP!7I@5B3=(^JHcY21cr+e^G{ts)BR z^?wnS^}lXL?*Q~P;^?;u?(HM}Y$rbIXM4sfwd3DH{J({GNLBw9;{Pqg1C0IOLVQE7 zgBWV^|0*H=UvKhP9H~7U{Y8#@UuFC0mwvXJFZ7f4`G3oo4MB(GZ`F$bAM)kD9?8p6 zP|rYqAHwYnvE9C0X8UZJOd804AshbrlP?KNeUenqgLkpqtE}zl^)lOo*UKO^{ilV( zKbFF+d`Af>%%7L%_{`N6J z-&FZ;ul3ZcTypN=NfybU3a@*x3jVsr_P#F~D&U%Fk*>p41`kDxh@W3=fxi{@0$^C< z-r{LF_R*HbS5)S=wXnFdr2BtUi{5L$s|5^BDM^2HRFE^->{CTl`U;X!&Drr1_Ug)K7 zephDSTmhZ@71Oxy^6#fma0N+m1`SQYi7mlYiB0X&yl!k4o0=Nhq~hUzm#kmX)r8TG zct^YibI;OS^c7z^9SM#k%VNf$777a|&q(Xh5DiPhMoO>b>Ks83rs99;HBk_fgp6td z*Q$lg>fp7EjNg3UCSA<#`#y6j-bha!tjI zV(B-2_lyP2lQIQwWny7XW&KoK6WV$|w^nAlQe0Wp@x7mYJ)d(@ONyjt^p@pIZLFv( z^m%fpFOLjYj$Pk)rm8vH<#t4`b%dd_@v)e)K71&$i#E% zGf*#-osb&uDait-U zL2ZfiQeO$)qT-(NDsywt-#H6Q5(|?nQAhNd7iGm4%T;D9K2^vfdTWy!mA_3gUadPw=^ObMTD9n!Yib z5Z~;YaK)Hr=4DjW7bKRIxc*tae)IjG^_6=@a-%dp7)z-?Fb5ObJkzfj|GHGKST}v1 z`@hg%Wx1*IN)iiN-6eVRdY}7;Ip?Y?Y%NVlU09Jgy}i`gGq?WV)~~S9ox7lSEdDsZ zdfITsZ~pt&MsU4XVf|unxIE{3`UUgn(*E$z>XuPc_AmPF|8@=Z?knwOGjif95?#G( zc|l_-_O8nM+Tc3FK7jXmt?u^V+;Ntbg_B&PZP9c1~|cql=~A$N#Q897cHcbfEMu*xsC+gtC%?cxgPOYtr|4Zm?W>>>>>> dd45418c (code changes for aap_38439 from ols) from pandas import DataFrame, concat, read_csv, read_parquet from tqdm import tqdm diff --git a/scripts/evaluation/utils/similarity_score_llm.py b/scripts/evaluation/utils/similarity_score_llm.py index 1dd07b92..4cb9e789 100644 --- a/scripts/evaluation/utils/similarity_score_llm.py +++ b/scripts/evaluation/utils/similarity_score_llm.py @@ -1,5 +1,6 @@ """Similarity check by LLM.""" +import logging from time import sleep from langchain_core.prompts.prompt import PromptTemplate @@ -7,6 +8,8 @@ from .constants import MAX_RETRY_ATTEMPTS, TIME_TO_BREATH from .prompts import ANSWER_SIMILARITY_PROMPT +logger = logging.getLogger(__name__) + class AnswerSimilarityScore: """Get similarity score generated by LLM.""" From 0a8cac06f3da0b123de3ab5aa5fd2f597c8fd8c0 Mon Sep 17 00:00:00 2001 From: Sumit Jaiswal Date: Fri, 24 Jan 2025 19:20:40 +0530 Subject: [PATCH 3/5] update relevancy if cond --- scripts/evaluation/utils/relevancy_score.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/evaluation/utils/relevancy_score.py b/scripts/evaluation/utils/relevancy_score.py index d3405458..a5ca0984 100644 --- a/scripts/evaluation/utils/relevancy_score.py +++ b/scripts/evaluation/utils/relevancy_score.py @@ -42,7 +42,7 @@ def get_score( # raise sleep(time_to_breath) - if out: + if out and isinstance(out, dict): valid_flag = out["Valid"] gen_questions = out["Question"] score = 0 From c562b2f77ba65c13e2c1746256633439919a0efc Mon Sep 17 00:00:00 2001 From: Sumit Jaiswal Date: Fri, 24 Jan 2025 19:38:35 +0530 Subject: [PATCH 4/5] fix files --- scripts/evaluation/olsconfig.yaml | 14 ++++---------- scripts/evaluation/utils/response.py | 2 ++ scripts/evaluation/utils/similarity_score_llm.py | 3 --- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/scripts/evaluation/olsconfig.yaml b/scripts/evaluation/olsconfig.yaml index 2ef06a6b..f2dc4565 100644 --- a/scripts/evaluation/olsconfig.yaml +++ b/scripts/evaluation/olsconfig.yaml @@ -23,18 +23,12 @@ llm_providers: credentials_path: ols_api_key.txt models: - name: granite3-8b - # - name: my_rhoai - # type: openai - # url: "https://granite3-1-8b-wisdom-model-staging.apps.stage2-west.v2dz.p1.openshiftapps.com/v1" - # credentials_path: rhoai_api_key.txt - # models: - # - name: granite3-1-8b ols_config: # max_workers: 1 reference_content: - product_docs_index_path: "./vector_db/vector_db/aap_product_docs/2.5" - product_docs_index_id: aap-product-docs-2_5 - embeddings_model_path: "./vector_db/embeddings_model" + # product_docs_index_path: "./vector_db/vector_db/aap_product_docs/2.5" + # product_docs_index_id: aap-product-docs-2_5 + # embeddings_model_path: "./vector_db/embeddings_model" conversation_cache: type: memory memory: @@ -59,4 +53,4 @@ dev_config: pyroscope_url: "https://pyroscope.pyroscope.svc.cluster.local:4040" # llm_params: # temperature_override: 0 - # k8s_auth_token: optional_token_when_no_available_kube_config \ No newline at end of file + # k8s_auth_token: optional_token_when_no_available_kube_config diff --git a/scripts/evaluation/utils/response.py b/scripts/evaluation/utils/response.py index f782a600..63c0612b 100644 --- a/scripts/evaluation/utils/response.py +++ b/scripts/evaluation/utils/response.py @@ -34,6 +34,7 @@ def get_model_response(query, provider, model, mode, api_client=None): provider_config = config.config.llm_providers.providers[provider] model_config = provider_config.models[model] llm = VANILLA_MODEL[provider_config.type](model, provider_config).load() + if mode == "ols_param": max_resp_tokens = model_config.parameters.max_tokens_for_response override_params = { @@ -49,5 +50,6 @@ def get_model_response(query, provider, model, mode, api_client=None): prompt, prompt_input = GeneratePrompt( query, rag_chunks, [], BASIC_PROMPT ).generate_prompt(model) + llm_chain = LLMChain(llm=llm, prompt=prompt, verbose=True) return llm_chain(inputs=prompt_input)["text"].strip() diff --git a/scripts/evaluation/utils/similarity_score_llm.py b/scripts/evaluation/utils/similarity_score_llm.py index 4cb9e789..1dd07b92 100644 --- a/scripts/evaluation/utils/similarity_score_llm.py +++ b/scripts/evaluation/utils/similarity_score_llm.py @@ -1,6 +1,5 @@ """Similarity check by LLM.""" -import logging from time import sleep from langchain_core.prompts.prompt import PromptTemplate @@ -8,8 +7,6 @@ from .constants import MAX_RETRY_ATTEMPTS, TIME_TO_BREATH from .prompts import ANSWER_SIMILARITY_PROMPT -logger = logging.getLogger(__name__) - class AnswerSimilarityScore: """Get similarity score generated by LLM.""" From e346349eee0d04b7ccc65de6a1828220127d4a37 Mon Sep 17 00:00:00 2001 From: Sumit Jaiswal Date: Mon, 27 Jan 2025 18:09:26 +0530 Subject: [PATCH 5/5] fix readme --- scripts/evaluation/README.md | 6 ++++++ scripts/evaluation/utils/constants.py | 2 ++ 2 files changed, 8 insertions(+) diff --git a/scripts/evaluation/README.md b/scripts/evaluation/README.md index 360db7d7..08877578 100644 --- a/scripts/evaluation/README.md +++ b/scripts/evaluation/README.md @@ -11,6 +11,7 @@ Currently we have 2 types of evaluations. - QnAs were generated from OCP docs by LLMs. It is possible that some of the questions/answers are not entirely correct. We are constantly trying to verify both Questions & Answers manually. If you find any QnA pair to be modified or removed, please create a PR. - OLS API should be ready/live with all the required provider+model configured. - It is possible that we want to run both consistency and model evaluation together. To avoid multiple API calls for same query, *model* evaluation first checks .csv file generated by *consistency* evaluation. If response is not present in csv file, then only we call API to get the response. +- User needs to install python `matplotlib`, and `rouge_score` before running the evaluation. ### e2e test case @@ -21,6 +22,11 @@ These evaluations are also part of **e2e test cases**. Currently *consistency* e python -m scripts.evaluation.driver ``` +### Sample run command +``` +OPENAI_API_KEY=IGNORED python -m scripts.evaluation.driver --qna_pool_file ./scripts/evaluation/eval_data/aap-sample.parquet --eval_provider_model_id my_rhoai+granite3-8b --eval_metrics answer_relevancy answer_similarity_llm cos_score rougeL_precision --eval_modes vanilla --judge_model granite3-8b --judge_provider my_rhoai3 --eval_query_ids qna1 +``` + ### Input Data/QnA pool [Json file](eval_data/question_answer_pair.json) diff --git a/scripts/evaluation/utils/constants.py b/scripts/evaluation/utils/constants.py index 48190618..c7784a5e 100644 --- a/scripts/evaluation/utils/constants.py +++ b/scripts/evaluation/utils/constants.py @@ -11,6 +11,8 @@ "azure_openai+gpt-4o": ("azure_openai", "gpt-4o"), "ollama+llama3.1:latest": ("ollama", "llama3.1:latest"), "ollama+mistral": ("ollama", "mistral"), + "my_rhoai+granite3-8b": ("my_rhoai", "granite3-8b"), + "my_rhoai3+granite3-1-8b": ("my_rhoai3", "granite3-1-8b"), } NON_LLM_EVALS = {