From 0a6829d2eeb084caf089aba0489a5ed96960fb94 Mon Sep 17 00:00:00 2001 From: Peter Staar Date: Fri, 20 Dec 2024 14:09:20 +0100 Subject: [PATCH] updated the README Signed-off-by: Peter Staar --- README.md | 23 ++++++++++++++---- docling_eval/cli/main.py | 10 +++++--- .../evaluation_DPBench_tableformer.pdf | Bin 0 -> 13889 bytes 3 files changed, 25 insertions(+), 8 deletions(-) create mode 100644 docs/evaluations/evaluation_DPBench_tableformer.pdf diff --git a/README.md b/README.md index 6aa2c00..104a01f 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ docling-eval % poetry run evaluate --help Usage: evaluate [OPTIONS] ╭─ Options ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│ * --task -t [create|evaluate] Evaluation task [default: None] [required] │ +│ * --task -t [create|evaluate|visualize] Evaluation task [default: None] [required] │ │ * --modality -m [end-to-end|layout|tableformer|codeformer] Evaluation modality [default: None] [required] │ │ * --benchmark -b [DPBench|OmniDcoBench|WordScape|PubLayNet|DocLayNet|Pub1M|PubTabNet|FinTabNet|WikiTabNet] Benchmark name [default: None] [required] │ │ * --input-dir -i PATH Input directory [default: None] [required] │ @@ -35,18 +35,31 @@ docling-eval % poetry run evaluate --help ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ``` -If you have DP-Bench locally, you can create the dataset, +## End to End examples + +### Table evaluations for DP-Bench + +👉 Create the dataset, ```sh -poetry run evaluate -t create -m end-to-end -b DPBench -i -o ./benchmarks/dpbench-tableformer +poetry run evaluate -t create -m tableformer -b DPBench -i -o ./benchmarks/dpbench-tableformer ``` -or, +👉 Evaluate the dataset, ```sh -poetry run evaluate -t create -m tableformer -b DPBench -i -o ./benchmarks/dpbench-tableformer +poetry run evaluate -t evaluate -m tableformer -b DPBench -i -o ./benchmarks/dpbench-tableformer ``` +👉 Visualise the dataset, + +```sh +poetry run evaluate -t visualize -m tableformer -b DPBench -i -o ./benchmarks/dpbench-tableformer +``` + +The final result can be visualised as, + +[DPBench_TEDS](./docs/evaluations/evaluation_DPBench_tableformer.pdf) ## Contributing diff --git a/docling_eval/cli/main.py b/docling_eval/cli/main.py index b36b8b0..fc30c7f 100644 --- a/docling_eval/cli/main.py +++ b/docling_eval/cli/main.py @@ -107,18 +107,22 @@ def visualise(modality:EvaluationModality, benchmark:BenchMarkNames, idir:Path, # Calculate bin widths bin_widths = [evaluation.TEDS.bins[i + 1] - evaluation.TEDS.bins[i] for i in range(len(evaluation.TEDS.bins) - 1)] - + bin_middle = [(evaluation.TEDS.bins[i + 1] + evaluation.TEDS.bins[i])/2.0 for i in range(len(evaluation.TEDS.bins) - 1)] + for i in range(len(evaluation.TEDS.bins)-1): logging.info(f"{i:02} [{evaluation.TEDS.bins[i]:.3f}, {evaluation.TEDS.bins[i+1]:.3f}]: {evaluation.TEDS.hist[i]}") # Plot histogram - plt.bar(evaluation.TEDS.bins[0:-1], evaluation.TEDS.hist, width=bin_widths, edgecolor="black") + plt.bar(bin_middle, evaluation.TEDS.hist, width=bin_widths, edgecolor="black") #width=(evaluation.TEDS.bins[1] - evaluation.TEDS.bins[0]), plt.xlabel("TEDS") plt.ylabel("Frequency") plt.title(f"benchmark: {benchmark.value}, modality: {modality.value}") - plt.show() + + figname = odir / f"evaluation_{benchmark.value}_{modality.value}.pdf" + logging.info(f"saving figure to {figname}") + plt.savefig(figname) case EvaluationModality.CODEFORMER: pass diff --git a/docs/evaluations/evaluation_DPBench_tableformer.pdf b/docs/evaluations/evaluation_DPBench_tableformer.pdf new file mode 100644 index 0000000000000000000000000000000000000000..2112e69d09b0160396a75d6eb2838c882d4e7824 GIT binary patch literal 13889 zcmb_@2|SeB8*tf8mh9PIBoSsG#+F^SFeHSGu@-|dByE{XYFp=e*}V=RE6qp68tR8AV%TQ#G_Y4xv~w09~p=pdl0# zB zc<%6~&>@Y5N0SJ;nLV9M_l9r_9kygT-J3>*@bDu7Y41n&@~8SjgkNuIK@@v$IGypKc*qajMU* zq8-X}tX!Kt51ViumAhlh-C{r$^3SN5982&Yvh1tK^PTJLo}q?{xNb%cxcO7!jLc=q^swQbqL;gdImg`4 z@8#Y@MYIdIF^%1|a}nz;6@=Nj`o|JHCnplKa{YvbLui}V&mD08Yi3Szdre`0z;@;L zF$zI;DC*3#6#^N^RitL}vxcMl^6-m@Rr?>upf8(r4Y%>@*dgAPl!xn&ZL)6S32!Ky zb?uTU*03^HTF)VJ_}!Sl^z-b7>+Z2fH#w=DNn#&7?JSteQ+2+;ayK=FcQq99Cbwn# z%D!%`ShPk>OW;dq1W}*gFH$1A;MIRpz#UeEvI>0t{W%34RtRBQ)0~|;|90r z^SQA1`Q}S{u~v}RUJrffyK8ye3hu6bY*E zO0TWqdeRdumvU}pjBsp?Ucn<&K*`M&cERuIKNp&Ji-RLdoy#mNp5zyOP~%a%!XzmV ztj~7W!}v;=PtTJD94C0YNr@Tx8R8ZN4MybBtv|V{ANj`j$*;8EU2x%n1ESP5-Q=S< z@(BRxLLrR_mv*V#)%!jGNkaNL>-U~dzhP+45#4eo?((Y*`=}>)1yQf!i?k@pIMQZg zZQY8qodx#N(`Y7-8ds;j*(DiX?seqG zQ&Jl8!^cjCcAN2_{p;d%Mr6NEOYgbn)vLPaVO_FnMaXxHGn|U0q+4&z`E1|oYaCFo z;N~neVKa_O>BsLn%zAD}AGv#dMF{6z^H)0lhv8v2d!nfoU?wNd`PJG7u_-(AhV@4^ zyGlMs{;<(pefEk(=*!UiLQ=M$ zk|2e21L(E8i8cS=pEOx@+9hJZ2V2a~$gbTE>nULt8w63!CsmcbygSitb-Y2eD zTMX&Pkvn$d>?>PECB&6#94~x+{^hh)zv&mz>*%N3-epo6zdUtO_;9fKCb?iTkpEsq zppA0U!POxHHr6+2Va67>Yc~DxSB!bQ`)5)!w`?(?y+3r)PS)W4_^ecH?QK`xAJ@Kx z&s5{|=LWwxlOC{)`C6O(b!q>nChE~V(ZFZLCg-s~_P(7&{%p zyF;XQN(d{|EoXj5kVZNtUvlhnZ0~2Of$bXa<|M>r=>fKpmT8wxJ2chODg?DN$J}Da zW6zy@G=}Xi8;?5sSA5CiqRj`SKi4U!HWoeKWWp&YTG!n!eO0wc>T&n$jR(B*$g^!LeL1 zDoS6s&5QQ6ZhohJx~rG8$IJ!o=oGsIY?JTnw%G1+TyT&`(oU|+3ZJ;FS<>t8G~`6^ ztXf%2PNk^7*pj)W_;^`Pa$fIkiop~AEIe=Ev*_v4V;dXs&(BOrjLR5umanH%yT0z=wsm*;e(-@`#Sf)kWHs7dHcptx#e`TU zepM6w-^ycsznZrm;$iE`Q`TX5Ij?7zA>UV*%qvh(o=;cG^+-TaL!5Xjw53YFKDIxx zZ&XGw%{Fw<*a@rn>|oLMZ|$h26Ha$4FKcWqqt%KINFDhTF$^34EKzF>!ha)%#cOOq z**OevH95EH9muIq z#tXl#sVVqa;*)TB3l@D)BU@0yLuD6@|MTZ!&58cBlz^?|u3U+tYp9OYIo;LvpOM5d`~TwiOgtm1IQjyXvdT`~X0Br8+CwX7j^ zJtLiw>Bmf{#mSBQK zX)I%Fq*+pd7c!dBi&I}+|E(|YW6}Dm6VDIyD=EG^EY&A!vLmy~%Gq8f1($yCz`Lp7 z^4w|dfKDHumo9}_iElDSxMn^f#66Gmb zVk1Ka-0v(J{ag+m-(|vTFgaauZx8om|MMMpuX0{E@jzSI=gNCJNu4T|ASZKo`uxt(xhL@+sAibr~G`L%w~D4%unv)%gfXrb&nc*EfY_#&9oB# zBU?7l*a~;wYP!v{`g4GZk#cD2ld+9)nv(1YnYCO=FkiIyjS8h@84Oyj8ceh~#WFS;u?7X6*)-xVNm` zEJ9{ef=1(s$Ax4udy%7E2%Xkp(%cI#Ldm2{_X?WQFN>XH1e_Nd#<*5&mpZqLh0YH!5&%YFEb2y}D@YUeZ$cNLPIQDQe*Kai6`BBy#~& z?B0?#zYZ2cS9-OtSWAwjjLFfhxD!{j&GA!FGp{RaX9yw3K8k&f+x2381K+-07QF*2 zEPuLjFkv2iEMCeyEatxz7jI`yy@TSZ{w^|CzoKWdMydI9!k778(mi1=eZ|U~ik=h& z79{#B@zB86H2j%$`+aF!=TvRt%obtaX_svA-pGfQt;uqHP}GvnbJv8R zt0RDUs<>JURm~Ch)aO}Zt=y|?3Z(m{nccmDpB|~O-LtQJ@h(O%@Y1)??+@p+^mZIF&RXyLGO?jyeHY*O ztqI@PTjc!Q28#nJ=S?kIyX$zZt<$(>jk$d*ag(Ay4|LwH?fuTbSJ;DQS^Im^~1Y+&u+@Q*vs zSq_aQt0fnx2&f(kfK5;FU9rnd5MmOq5j2;BP=zf(NMmak9*5h4!e9V$7`sjAVX${N zv7xs-Vu&7@i7QjcTx%O7Xg57vLhH4ltG1;?S8-SW6(NO!zD#go5>8>T%W;=B#56WJ zp7z~wdZ;aH`t|FIiz^9FXAe}RWp7TIl56W0lFp9ER#3R&)7tOi1^K+Rw`#8YQo%2D zapt9re8!PgdS z*uwGMA6O)42J>!%Uu8_HM-?GSZqXh)&&gA74sO34PM5j9UNF=!$3R_3?*2QKbE|V~ zORk+cO|lEVpjSLvl7HaMc2~7O`GsW?-VvZ&?O@h^dOS|5Pkd{Ayr6A z`V@8}S%Y%4-UQ8Y>K*-@F6TPp_eR9QP)YA`$4yF!7v+zRVUH^3sqD#*5LKPHOW3BA zraKJ1B9Ri?&gN&_+ZcE+1EJ;fkWE$a)=MuTYt8m*b5fFc24+m=T)6zc5}mtev_`$G zACxF$_jTL!XMdFnzOnt8*XHA`beWE?rz#%A5=E+Jh7t*!*2&LC$7h86Rgk1=X_wsn zVS>j8*Nm(Y%vQ#w2)mwsGn+Ch(CvrQ4PIUcyGSa#TZ;!`&QYog=#0B zBWNH%9!V8BUu2u_-1x$7^8E~keMeBz*;N#&PuvykhJl;zu#D-iFdolvC`~!)yH==9 zWYE`B%46i=)s#@PUUV(W4-1$n0gmbC^Jx^sSo@ z=+UyrdxEVh-tWKr+2$v`-{pQ-4UMZ%|Bbc5R5f3jY5AQy7`@zN<;X(SD-t<1rq~AZ zYJr}arn{^XKWH_pam733E1cv{UiUOkJ{%CLdz-Dt>O(WJr%9+W;6rXt!Bs3)MK0!9 zRs4?k`r+4VKL%IkZ^Vuk1~QQ;lb{=oT}E7IeawK570t!^VRu}9~cze}22= z)9~(9GHp=-b{x@)2@wgBuO9aEzc3+)_zs=gPEz)iJ7E)?kFu9aR^ByDITjU?u|u`k zFT&mOfpwFZ+Ablrv!z+-;z2hjvu4bW2d46xbgHc~>s?>)@Aw z%U|YJJP5#EC`uQ)?Sw=MPpo@T|+>RA%=s|5WoW$^%7$yXX0Q*QK1_{g9G7 zvx@RvL5B&JOk!B{at{6`DXk&wAcp;Lkwp;|h4@_3`jP+r&&%r{O1_WoHUqky`SC9ce?C z^cD7MU2RRTti=AHyUUKteGr;gQ5m6k7qw(KaUd zFp0x7ml0Ozb`S-lP+aU?dG>P(2k*+ygh(BUn0G`yo?`8G-x0X*5H{S0Q%jK_@79V7S8k2|zw~V6=Q&wfJ zc2pO$QswMmd6*fDkk4y4&avW_FRv04uuS3_41O7bjY+B{jSb>5p2>?*h?*PLNw@Gu z-HA)l?{*O#wI$hwP1c`i+cH|ta!K3%PWH~k z9q%dL!#BINIhrLOxDhXSg`7zWa|lxjw(30o*=}$)+&N>P8J{=eX-TM5z{`7qO>2}) zPnJ&d%_KrLO7>>W6Hwn2r&991Ym z!bhS#f_4jR>RWv`Rc?)ROn9#A{Og|C+w&pu+MUu%BLm9k zM|P^a^onaePsg?npZgyF;Fd>Lb*$Bg*^ND&-+r>nDEoY4!b&D#8Je&R?J`Nx7G#y< zG1{}aK;f$bt6rxMMwu`ZM0pxOs_%A!VylH>9zTmo9F0k+R2*dgP|D^@nR`c|;suA< z!lE3DhWs)}`))P+AE*E2M+n`ps#HkYhc7bYjQHu*S|ZOU3WsUuCLhci1%KZDxz2Nt zcrr1%?VdE({W1H2S%P+9RaM*1qSk?u-u)ZKw~+mvE!(+o@;O$Scamr z{d29onRobh$G@R^5foKSkm*+xXSxu7?Mr+sJTv$t^O_MZ%a6E!6;|bAtzWx zsOt1U6ZRW#Af44QL8g2g)K%osTM5xx`Pc9?WD6aU=drNnm@lIp%XdE_sn_(Ek#K4n z-&_2=(~s#;-plsKlv1fdwXfYzUEpICkV?y`QE+oAR~HoU#~WC8*1M=hRo`&`DOsUV z#iRcv@lHq%>1aXS&fv41$Tv#%PnocYNzAkyTVs+0E1zdVIKCk0l ztw_D8G^pz;&7G}G#Wopn=MR*y45-r4FtkrN9CKl3;#kI|>~kIVs@7c}pF5YmUGs9!7LCU*J{@vrNwtgF zLXPH3(Z1>NwQuLT0J~N4=y+Rflz$^4^rj%1O+4wk@0tu1p3P4>RknUd7JENcxe|fO zmEU+Z$FF#Ml~lxgWlqDV1IMc)<*d0TlQ}3Is;;{(T+Lk}Wkt_UOmeE>ZS9uh;dRg9 zC<*RKJ^__WDxBqa=M?Y0vdcs!?8chEYPYpyR6pt5XSeUI+k`AvgQU{eJuB}?y16sq z9n;WZ88^cwDMUdKv9X2Tj#1j=8Z%SUe!bz_`~}V)v%+^QIH@X8>wA&HetPf9O$(~J zc8;oAq!~Aerb*itQx0jb-(6+V#YzLKVq*oRG&5d<^E>G?nCkH3b zCBEZun6nvk&6u)|y_|;rw#K1GJ2uJutGmg73wq+GF{)hr3wMyoI`)H!_}A(TnjfCYE;|M|5E4~`q`Xs-TF<(v9vr)(a4ym;|QEjVrn-4?Mz zc?>BdogZ1&K342M+cdmQc1E$>@cFTcoDNd+jOaDtPPNNsodP%15Nvirx=z_h5&6fR zpGLLL2V3MvZ{@lnELjt4z9M$VS5BF;Sjcunzops}Lt>3t`!v))ge#G@F7ZH7P2LNo z(xQS-y3X0{FVZc~c1c@r9$nddui5;xoB8bN>=PEoX<6_0l^cGFuTRs_GefsUjS~_B z3)$C=$edN|xFf7}rIn?|C+EqQQIAHM1m&cytFE1w37Fm{bi7EqbV$4p^}HQr?^*Eb z3j5=?p58H&Z-(%f8xKv`rMoo+PiEfV#r4qdr=;k;mlTn-D6`^nL0*5&`a|`3?wZZ+ zABBi~^@k|AIjTGqzjeitM-%J2@~)jFx3tQ*eE!Jq@NRDc6G1SEmzHDY8HOUNN!SQ{ z<&b{MN@B7ylW+%(U4|VnIcCYn2EKpIoGLt8-zqy2f_WDgIzJJWOBJH%#yLdaiYd@l z-^rRQA}c3e`PYpLnMX~W^&BEej{I#e6G~dzb|10v7%+rt)+#TsXS9p9%_~(+82S%G;p~Zb~^m;#DJ??f{ zH`}1C#PyD%keJmIvVt$)#%zpwMX0U@G-B$MMNXkQN?-+#;^H>R#a%pE^nlGkFsaNm znLrc%einI*9=|2|JbFA-EpU6MwdT&Qai@~$;S3qK5OpOnWQo+L&jeO6JO7et{Y~{h zsQjNpucp6konzhZvF#8O9x@A` zG?sCGvraMRGXmk$%&A7P^9Ql@BRX{#tNcFHl^St|Mb8aG!8$r!A#2s?UZY7zH=L9j zs=o$NUO&{FZ@Z~?2W>D|sOFRv-rCn87}|(Ivs9Y4_30GW^S8Qk33_G8*49JvZpUUF zKRTy>>b#*nc&w_zXS;4;ed0J*^OcobqkdGiMq4({x_1v}sVGu}5~icaIxqet8_y(?LNDjrm+r9!z9tlx(T7>YynfMB<$}F7 z#XAxv=PxT*RlcdTOq5 zm8pZ_17jLBWw!S-~U;6 zKk1>POoMBDMTT7VG%5YO+#7qn$ZYjfRRg^pJ8&bJ6>6bTH!Ew9rD^qvK8Io({7>9m znWMAPsQ2M{J0mffuE%HX*IyZ6(>r{F%0EcC9-UPwHt`}D|xPQS!NHnlhsR&;O zqQd7{6N7i{&$eW)iRM`8s43Mu+Biw%_ZD(ifxWU+a2iY;qo=gpe-5)KKZsbR%fyqcK@~<`qk4GTkImX_h6q}hk4Q&S%5GMU|lzp13@mG;nJlS<{K{L qN7&N4Y14kFM2Dm^$V}M-- zNpvs>YcieY&q%UUN1oWF#(jA&|!Y zK0e+cuMbY-+XlfOL_&kf6p&*Gwml^%Fpvxq`5>LOn{JH5E3lz19Ag#2&5l`^am~QekX(sfRKR@k_x~>Xf&7`0wDpT z0i8%XgbV|00``74NSOgyfyk}?UUWaegoR|vf3htZa#{Qk^pD(slN^K1|3gdw_W-hQ zC}7mG1ZEIQfm0@l7;x1MLx3ODFz^+h5jl9%0-I0>q|;&(BxHhAQGbd7)i=N!Rxx|J zcc3HOuqH<^C;|lGFu!MOE=Kd)AOZ^Y|Dz!OU)RBcxC94O5@db?$!bCbECE3T##I9d z8>4}TutXH2-~pcSdw4vSfL^Gv8u0l4TR|+o$AYnNf!A0pc<}KBje;4Vi30^JMAU>d zK+XmMs5Bk`*MJL9Rtyo300ko82@oC+76E%q1L(DeCO`lM+HeiO$ASVk?AvBnQ00+QW1Pl`o<}jGDh#Xuq z^aqB7gC9^Zi(s^2(%>2thFoB93k87(Br;~g{Qai^5L{RS11N^yz*qsy7u8@1fgasXd_2A!y z!E2xwuul9|82SL~#A0FS$}gRO_2Ksd27^H20Wa za2EUr<`*n4z~6;6ydlQ-K(OwD1q32vtm6awpM@6eQGm~5Joteu@`aW^*h+8<0h|HQ zb;jEO2$%v!D-Z&*VYH}V>nyZ_fD-*`0gnZOI>zipclHZf2n4P_7!zr*-($33SBCpV z-!G5&3m_c=m%@x?VE+fygfTu00-rM(t=&LB7g`Kg32t@Z?tdK6qQQo}u_@5{1t9;t zhY?uxPX_4Uo%Y{906-cUfI&zK8A8JgBMmVBa~SZ5zlQyW0u1sOh=1VzrYrw+!j%3o z5@y>Xh6^5a;fD=KOMfpoiU3;yCYh1-9|k-g1Ib_cuD~q)Gua4a?!&RtdJxnE2L#ar z{5_#?tU4M5G^&svogN&jg+vDa{Z2iI=BtcAg6PI8i~@cM@cXo2FCWO0OxX#*EG_|t zG647oQH|lBO{_Q8!k{oXGzNo0<4{;EZUYLVghDAP|HhdJnzs)E*fJ~v7`tD8z*pdb z-uXa(_rZ}B{0C8g^`S9%Sf!WrVPPFz+6SVbrF}R8VDpl3IL$xD;ZZ=v{xuFf5P&e3 z^a0YA_Mr*zMqe@xgIWp$g~kK>`0sIOlqT?5OZy1G_Ac#1|49d$2ybrq?{8kAz>WML zxS)w3>2t|A3t4FX_Xg|G*h&!Jjy{6@EF+K{(C(<0XW~KeVW8S@FOtM1K^7OSdV}M;rx>I z2n6^X@o%^U4cHz0yH5iJ+p;Bn7~mh5_Tm4~7nFtuY|WR<)%=rJnrQHGa>+PN9O@4p z*VOn!Pf?oSZ|Phj3LKx7tVhJb?(g5Y5D9EC?grcH}jVa>)09-gR{Qv*} literal 0 HcmV?d00001