From ae6025c58632f47bda2b779e107f885422b4e2a7 Mon Sep 17 00:00:00 2001
From: sahilavaran <139779393+sahilavaran@users.noreply.github.com>
Date: Mon, 23 Dec 2024 16:55:33 +0000
Subject: [PATCH 01/24] Update SDXL README.md, improved CM commands
---
text_to_image/README.md | 34 +++++++++++++++++++++++++++-------
1 file changed, 27 insertions(+), 7 deletions(-)
diff --git a/text_to_image/README.md b/text_to_image/README.md
index 57c4343b1..657e7c969 100644
--- a/text_to_image/README.md
+++ b/text_to_image/README.md
@@ -1,9 +1,11 @@
# MLPerf™ Inference Benchmarks for Text to Image
-This is the reference implementation for MLPerf Inference text to image.
+## Automated command to run the benchmark via MLCommons CM
Please see the [new docs site](https://docs.mlcommons.org/inference/benchmarks/text_to_image/sdxl) for an automated way to run this benchmark across different available implementations and do an end-to-end submission with or without docker.
+You can also do `pip install cm4mlops` and then use `cm` commands for downloading the model and datasets using the commands given in the later sections.
+
## Supported Models
| model | accuracy | dataset | model source | precision | notes |
@@ -53,10 +55,8 @@ We host two checkpoints (fp32 and fp16) that are a snapshot of the [Hugging Face
The following MLCommons CM commands can be used to programmatically download the model checkpoints.
```
-pip install cmind
-cm pull repo mlcommons@ck
-cm run script --tags=get,ml-model,sdxl,_fp16,_rclone -j
-cm run script --tags=get,ml-model,sdxl,_fp32,_rclone -j
+cm run script --tags=get,ml-model,sdxl,_fp16,_rclone --outdirname=$MODEL_PATH
+cm run script --tags=get,ml-model,sdxl,_fp32,_rclone --outdirname-$MODEL_PATH
```
#### Manual method
@@ -95,7 +95,17 @@ mv <path_to_download>/stable_diffusion_fp16.zip .
unzip stable_diffusion_fp16.zip
```
-### Download dataset
+### Download validation dataset
+
+#### CM METHOD
+The following MLCommons CM commands can be used to programmatically download the validation dataset.
+
+```
+cm run script --tags=get,dataset,coco2014,_validation,_full --outdirname=coco2014
+```
+
+
+#### MANUAL METHOD
```bash
cd $SD_FOLDER/tools
./download-coco-2014.sh -n <number_of_workers>
@@ -107,7 +117,17 @@ cd $SD_FOLDER/tools
```
If the file [captions.tsv](coco2014/captions/captions.tsv) can be found in the script, it will be used to download the target dataset subset, otherwise it will be generated. We recommend you to have this file for consistency.
-#### Calibration dataset
+### Download Calibration dataset
+
+#### CM METHOD
+The following MLCommons CM commands can be used to programmatically download the calibration dataset.
+
+```
+cm run script --tags=get,dataset,coco2014,_calibration --outdirname=coco2014
+```
+
+
+#### MANUAL METHOD
We provide a script to download the calibration captions and images. To download only the captions:
```bash
From 6b567a7d3366d9161239ec322f29135016eeb790 Mon Sep 17 00:00:00 2001
From: sahilavaran <139779393+sahilavaran@users.noreply.github.com>
Date: Mon, 23 Dec 2024 17:03:51 +0000
Subject: [PATCH 02/24] Update README.md | Fix SDXL model download path
---
text_to_image/README.md | 16 +++-------------
1 file changed, 3 insertions(+), 13 deletions(-)
diff --git a/text_to_image/README.md b/text_to_image/README.md
index 657e7c969..ca28beb67 100644
--- a/text_to_image/README.md
+++ b/text_to_image/README.md
@@ -72,6 +72,9 @@ Once Rclone is installed, run the following command to authenticate with the buc
rclone config create mlc-inference s3 provider=Cloudflare access_key_id=f65ba5eef400db161ea49967de89f47b secret_access_key=fbea333914c292b854f14d3fe232bad6c5407bf0ab1bebf78833c2b359bdfd2b endpoint=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com
```
You can then navigate in the terminal to your desired download directory and run the following commands to download the checkpoints:
+```
+cd $MODEL_PATH
+```
**`fp32`**
```
@@ -82,19 +85,6 @@ rclone copy mlc-inference:mlcommons-inference-wg-public/stable_diffusion_fp32 ./
rclone copy mlc-inference:mlcommons-inference-wg-public/stable_diffusion_fp16 ./stable_diffusion_fp16 -P
```
-#### Move to model path
-
-```bash
-mkdir $MODEL_PATH
-cd $MODEL_PATH
-# For fp32
-mv <path_to_download>/stable_diffusion_fp32.zip .
-unzip stable_diffusion_fp32.zip
-# For fp16
-mv <path_to_download>/stable_diffusion_fp16.zip .
-unzip stable_diffusion_fp16.zip
-```
-
### Download validation dataset
#### CM METHOD
From 8ad28848f10cacbad8b615bbf2c237593d9c8b83 Mon Sep 17 00:00:00 2001
From: sahilavaran <139779393+sahilavaran@users.noreply.github.com>
Date: Mon, 23 Dec 2024 17:07:57 +0000
Subject: [PATCH 03/24] Update README.md | Added cm command for downloading
coco2014 size.50
---
text_to_image/README.md | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/text_to_image/README.md b/text_to_image/README.md
index ca28beb67..70b865204 100644
--- a/text_to_image/README.md
+++ b/text_to_image/README.md
@@ -94,6 +94,11 @@ The following MLCommons CM commands can be used to programmatically download the
cm run script --tags=get,dataset,coco2014,_validation,_full --outdirname=coco2014
```
+For debugging you can download only a part of all the images in the dataset
+```
+cm run script --tags=get,dataset,coco2014,_validation,_size.50 --outdirname=coco2014
+```
+
#### MANUAL METHOD
```bash
From 469a3bc71baca2cce30c7f03000fee0c906b4fa4 Mon Sep 17 00:00:00 2001
From: sahilavaran <139779393+sahilavaran@users.noreply.github.com>
Date: Mon, 23 Dec 2024 17:48:18 +0000
Subject: [PATCH 04/24] Update README.md | Fix SDXL calibration download
command
---
text_to_image/README.md | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/text_to_image/README.md b/text_to_image/README.md
index 70b865204..d4eb1ed60 100644
--- a/text_to_image/README.md
+++ b/text_to_image/README.md
@@ -127,9 +127,10 @@ cm run script --tags=get,dataset,coco2014,_calibration --outdirname=coco2014
We provide a script to download the calibration captions and images. To download only the captions:
```bash
cd $SD_FOLDER/tools
-./download-coco-2014-calibration.sh
+./download-coco-2014-calibration.sh -n <number_of_workers>
```
-To download only the captions and images:
+
+To download both the captions and images:
```bash
cd $SD_FOLDER/tools
./download-coco-2014-calibration.sh -i -n <number_of_workers>
From 41bded9944553080a0d19ab13ceac78018ddbcc2 Mon Sep 17 00:00:00 2001
From: sahilavaran <139779393+sahilavaran@users.noreply.github.com>
Date: Mon, 23 Dec 2024 18:08:58 +0000
Subject: [PATCH 05/24] Update SDXL README.md
---
text_to_image/README.md | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/text_to_image/README.md b/text_to_image/README.md
index d4eb1ed60..beeeaf6cc 100644
--- a/text_to_image/README.md
+++ b/text_to_image/README.md
@@ -78,11 +78,11 @@ cd $MODEL_PATH
**`fp32`**
```
-rclone copy mlc-inference:mlcommons-inference-wg-public/stable_diffusion_fp32 ./stable_diffusion_fp32 -P
+rclone copy mlc-inference:mlcommons-inference-wg-public/stable_diffusion_fp32 $MODEL_PATH -P
```
**`fp16`**
```
-rclone copy mlc-inference:mlcommons-inference-wg-public/stable_diffusion_fp16 ./stable_diffusion_fp16 -P
+rclone copy mlc-inference:mlcommons-inference-wg-public/stable_diffusion_fp16 $MODEL_PATH -P
```
### Download validation dataset
@@ -112,7 +112,7 @@ cd $SD_FOLDER/tools
```
If the file [captions.tsv](coco2014/captions/captions.tsv) can be found in the script, it will be used to download the target dataset subset, otherwise it will be generated. We recommend you to have this file for consistency.
-### Download Calibration dataset
+### Download Calibration dataset (only if you are doing quantization)
#### CM METHOD
The following MLCommons CM commands can be used to programmatically download the calibration dataset.
From 6df318e51636492cdfe0a3aad1c832aaae997abc Mon Sep 17 00:00:00 2001
From: sahilavaran <139779393+sahilavaran@users.noreply.github.com>
Date: Mon, 23 Dec 2024 18:15:25 +0000
Subject: [PATCH 06/24] Update README.md
---
text_to_image/README.md | 2 ++
1 file changed, 2 insertions(+)
diff --git a/text_to_image/README.md b/text_to_image/README.md
index beeeaf6cc..84c8c7245 100644
--- a/text_to_image/README.md
+++ b/text_to_image/README.md
@@ -56,6 +56,8 @@ The following MLCommons CM commands can be used to programmatically download the
```
cm run script --tags=get,ml-model,sdxl,_fp16,_rclone --outdirname=$MODEL_PATH
+```
+```
cm run script --tags=get,ml-model,sdxl,_fp32,_rclone --outdirname-$MODEL_PATH
```
#### Manual method
From 1a2f391ecbaaa46ba6d6dbf424ba9c8c0cfa2f45 Mon Sep 17 00:00:00 2001
From: sahilavaran <139779393+sahilavaran@users.noreply.github.com>
Date: Tue, 31 Dec 2024 07:50:06 +0000
Subject: [PATCH 07/24] Update README.md| Added outdirname for the bert
---
language/bert/README.md | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/language/bert/README.md b/language/bert/README.md
index 6e0468666..fde646755 100644
--- a/language/bert/README.md
+++ b/language/bert/README.md
@@ -55,7 +55,8 @@ The below CM command will launch the SUT server
```
cm run script --tags=generate-run-cmds,inference --model=bert-99 --backend=pytorch \
---mode=performance --device=cuda --quiet --test_query_count=1000 --network=sut
+--mode=performance --device=cuda --quiet --test_query_count=1000 --network=sut \
+--outdirname=results/bert-99-performance
```
Once the SUT server is launched, the below command can be run on the loadgen node to do issue queries to the SUT nodes. In this command `-sut_servers` has just the localhost address - it can be changed to a comma-separated list of any hostname/IP in the network.
@@ -64,7 +65,8 @@ Once the SUT server is launched, the below command can be run on the loadgen nod
```
cm run script --tags=generate-run-cmds,inference --model=bert-99 --backend=pytorch --rerun \
--mode=performance --device=cuda --quiet --test_query_count=1000 \
---sut_servers,=http://localhost:8000 --network=lon
+--sut_servers,=http://localhost:8000 --network=lon \
+--outdirname=results/bert-99-performance-lon
```
If you are not using CM, just add `--network=lon` along with your normal run command on the SUT side.
From 5b9b622392860d63674be6058a9577ddc8f219e4 Mon Sep 17 00:00:00 2001
From: sahilavaran <sahil.avaran321@gmail.com>
Date: Sun, 5 Jan 2025 15:12:12 +0000
Subject: [PATCH 08/24] Fixed X and Y axis in coco.py
---
plot.png | Bin 0 -> 21916 bytes
tools/upscale_coco/coco.py | 6 +++++-
2 files changed, 5 insertions(+), 1 deletion(-)
create mode 100644 plot.png
diff --git a/plot.png b/plot.png
new file mode 100644
index 0000000000000000000000000000000000000000..d2a2a58ca3bfbc0ac4755961995b0c60887e6ad8
GIT binary patch
literal 21916
zcmeIaXH-<%wl%sCGa}n<KtaGj5D-v_k`0Ijl9OZ=Nkx_<MKpm5h!#mQNCp8FkQ@{R
z$r&W0h(IBkB2#>0;y(B8bM8I&yYIF3{=FZ&?WU_-YtFgm8l(3<`j}6Y6=i96GwjAN
zj23tGvMPpc@xm~w!JXUTZv>lrf589FJIGvjP_r>{aK3rR7*n|EU~6gPU}=8qn3M4x
zdvhCWeqIq?fz!v#92{)zMfv!w{^JL{Hg`<<SlCWF!cBJBUe&h8FuI%QKdKa|WOEGD
z*oC`%LH%yzM4$6rb;o|njB5+~an`tG={<3eqE9?MCVuEdY|0xhw!v4KnMJ2hJ%0Q>
z{(#-KeKb3TF5mRzzN~V&bITTn(+^MJjyYVUq56~c`L*3)LFdb)+#&^Awy=hU)vY?}
z+h>f_jYPVYx}FP*n3J%tynWdFI1DGBmQGd}{_oW-gKfnyTmG%m@F(F*|NkHUzgYu=
zXV&GxU6w*jVouknsHo0`U3+G|1H;ts-8{^7>Xh>J>o3K{d*}2HQ%&za3P1Vw?b|bJ
zY7Eo!J}LVwMaK8gGwX`uJH51y9Xlr7l2s;s@7}$=ldwhAAAP>1a`$fjx_d9vJ}j%;
zrnM9<p?9h0CuUeW9KC-4UaPeqwiCk!O?G;rKY0J|{!nSWWw1G3)^Vz2G0kqMa&~S`
zEtFS(ey&l<&^96=f%Ws}&()Sf=?W1-Iyq*|+JbW<jas!KJj3JT>63j$7OhDNf&yFM
zc_V{=Qccf~wH8uV#)l^+GP24jI%3X~ZMA%5uf1s*qn#rj?4ZM!dr=jRB}D{<ge1Yk
z#^QXKThcVre+ja9EX7p@uw-$nC7ohplWA^lCUs`TC`O8?Fjm0bZFdIVJnS7Gk^eF|
znjj~&J|F0@^v#FMX6{Fz5(%ErwlL8%F%FMODlQh!&?`_Sww1U#>)Y20h6~$7vsRmA
zbBc@W1&Gf+jwX;&eJOi<Kfj8NJvKZtGN)e-%gbMJ2LEG~;IiTLg5N8~lTuLVIG*(3
z!H#t8Y?G({{_Xd-(eUOg+`04a`_K?~#yAX(7#PTgOH54cerGJ7$UsL&_qnR7$Aa<&
zpIdY5tvUg|I8tc1i%wp{@&zBoL+1;FMXTAi4Bn&OG5<5dCe5nny-qGZ_=-P6qAZKM
zx-UJS<djfXSJ&>ndoy!*MMSP2@|7`dh-|J7<j_!2v1<tb_3M}Yq<y8|k=?s@zmxh{
zQIX|li(%yrXZG+cQM|U71;6g=+7`Y{wwBFP=Wsb+crWcumnEqqBf{YBw{198OUu{q
zZ3Z&)`U&!jea?m%<g?~2to|%g{iT*~ukXAED-hT!lNkQSGQDGxLe^R9DST-5vrnI?
zSEKL8pHycA1k_<ezs||L<g<4Lmb{@z;p)}LckbNb%BZcbR=IH_c5Q9#$JiLBl2W66
z$HRvYBkf~Mvb9aJm(R&kVOYb+$fe}@(H3>4TD}(np+3<iCGnFJ9vc37)H~_Jwv@t-
z$u(_^mK$G~>`Q<A`0<=|YrHI7hJMlemcsUQt#sJ{W+E#K3yZ!xxjh(fDHPF>sc(@5
zHx{fl;h(>v>g`>ceW7Yr{y^6!!x&0@GQ~}Q>$j}!1(z|mUvH~&ZdPnBnf^#q^W{qu
zkxhQSL2SxwezbWZ92dxL{^QGwh2gS|^(j%?K`(a2$dtY!=W&1G{w8=W302Ilbq4$%
zYj=6BzmbM_1dlBez_*<`c<|uZy&cRc?K4!^;Ilh48H5|nCiOZ6vIm7OQ5$`nkC7#$
zs$G8`ubV!ncWd`jxJ71ja*RhBJa4REBwFqE?F0kY2|@qEXFnR)uPxabo0xQmo;-Py
zJRE9BdRxWZ8hPHn?Uf8o$?9Yg`U*-+Mw{YR;G%%b>}_LHQ*L(lE8JR{;^Pk>!p9P<
z#Gj|j?pF&odRy~rIL_SJsin!{<N0wddA;wFUJXxIcT!=t=3JKE{Y$);vNF54BaY`<
zld_gies7aZSm>Aiz2{*sk)uwXYBpj?8cX)m!?pC;@d&spbF|MLf7T{MVSyMXIGOAh
z$Pn=|De#fzvWqKzCAOh(gai?*)@mO{g-QFTzBv+}?G$6M;S{sy5}sbqaL%VH)ymc@
zUf13Ng(~HfcTTer9L&>%n8n#}3{DF9NxjWD56S4glY2I=PjpPI%M3|fQ?ph{U2_*F
zf7I6(IFnQm-}7<fIJO1z!mAnFY^_aSB-JOZ6BG8v-3ylz*USj-aJDsFlGt(#!%_}a
z%{GiSC#J3}Pn+XfN;>ruO}9G)WeKa5Hf)B&%&&Px_5zgnj2doUHTUdLlm0%LJZD;a
z2VD_5bc1|L<AXvaIrxD7Wr`PTs{GusdPe5-$CVTqBK<{_G(*|O9o|-+#%XlF5XZHw
zOX%WTtR{>XZq;;lp;ahn)6G4pt8}`DOCDFS#l1veJlc#rb@cn_du5r;IT|Mp3)L?X
zLi;zyp0jd_7T^2H_KB-lMWRZ`TYmEa8pJT&*JtI`a(K86iSwQPF9@#;U6=2=h~)F@
zaczz<C95T1ygDE+wOl82JGoeJHQMa6^ByM$-TZ6l5w-?w#s^nMLW|}%gpP)=>N7Pf
zXK{SudVMeEfmlWGF|?5OqIch2jhDLG+%R+q*OJ1hYq7>N@nfPoC@cLWx}NP8;-s#`
zr=+D~{)l0Dbu{>RbK%*vu=t*fg6KtHMZ|Zz6;>Pf^7J`eCOTN%K3UInDE;{{!yV4v
zq|0cPhzWlCA2F<7c)IZ8z?a*^%HFmsMyol&^lGJ!d#V1wu+cysZGy>Xrzp;J<-iq_
z5+Q*rCEY@H4!SAF(G+^G+nE=zd%nNKVpC0IC0q7g&!Y8J$;Oi0#?PT8u*Tac-n-)Y
z{FOV$?%|un$_4Y}v%+~Vl6|V1Gvd6_d$8sAzBw8n6JS;|BBritqLz(i|J>uN5zyh!
z)W?9|g~lMPYR1#aACqJu(|s1FI`v**p44<$FxAG%)-*+7a}=K%j1Kv06JJqt`2HBX
z*)rtbGd8u+s>6J3xsVAB!!h#a;guUkNztpflI3cxcGJ$b_ddL?-}>%FyzYYsf50$O
zDt61&%(u-txED<1vyS~-s26ax&4DF_w<+=`jKWKlOM=_bC?sjr*7|L+__gq_QgU=;
z{O2G(l*HQvy>FV$tXv{|OYh^#c$4ZvEma~Zu4$(w=-^;s`zIPC>G>MJF=2MdN~hqY
zF3~~0>PCGjht36ZF;nl6NHk9`(GwF_qhZIYbd2%OE(n;nWH8N_lX;BKogMbA(fi&L
zm8=-W6<Do4)?DhoI@d99cgWwgJ>nWW?cp=!2KF24i!o%k4a&NOkqD#-?us+>lXN>A
z8Y9Mo9asD^@#;!3ex%g0`KPRF-y-#X?zsVX%Q2~ESbZ1TxvXng>Q*2pFF#RRP*Cvn
z*|Uzp4-dH8$x{P{Q)L^a?p>BY*<WH<Cs9i;ZCCAke0}DAV!~gk=iq2<5MM*1@s`-o
z(9nw)FQ%#Yq^Jz9GO)3;rx}zGOt(pAcj*ngZ@~sF-yfAslnqHBUH5q6RxCdMYl6wq
zW-(o4+{gFM6V4mB?mU~Aj~_p_rD<?O$h{aA7N*hf^zGv#1|iFrYwPRW8A@_;s@W#>
z2M!)gREfXBDJ;BrfAy_S#=GmMgPGwS2u0L4>5ea0eCc6yI9+!6g~A$nOo~8i?3DR~
zqRhr#ab}OKu-#ch&-L8hG&GPJ5{Ie+EV^^dMG5mMDCv56dC^!GnYXe#PW9_E3R}13
zb{be1ZuDq>`10lI#6X#$lGJ(*#QCfrKW@7a2+{V2YRQW2Ip(bmIx#UZS>@&B+>ES1
zBA)A-wGY+4=Q3~PBPNK&SIk3rR3G?tE|9R-^zHQn2M+w`m>S^Shi#nV<5SwVZy#Ye
zSZjXp0iD5{1sl)R2mma(06S<pm_%%)Gj#Jh!UZkbl9giYJFaJFYG-J!zWWU^bvL$9
zVfe;M>H>nmS&j3D((4SE!wIGwv_<TZ*&d;fvC)g27{%bhcG|W~eKF(PxAop+rleeb
z^5h9&vS5U_uu(HZClyj1q+1aiOdz7(O?fM(JpSJ&pidGNG?|gLdelp<j~+ehF7qtA
zc<B<~^*qb2b1<Yn*QpZvK2uH2kURGFe1%FFhHvHDrrEdH&t6iz(mCy{rDyLhnMD7M
zPU{nZtB1Sx=UDfNGKo4|1%%caLz(A*_2fE!`~p$n&X21K;R0i?Cl2xIrCfNhJr&Sr
za0mUNL!>7h(M<px9Y&ju1LDx6-mzCbLddEu%dkx6$bJl?&z2lYu#Q-L;87>SDP^Nn
z;-I8ea*p*1MQ_T7`^TSU7qqpt<*r^;wzkd;7qUWI)4al)7T=wd%F4;lug1(Aoc$|W
z!gYQqK#CM$Q+nRQGdeoD`@L1fAwGlG09IkzsWt<p;gas6YQ-4llm6c((dkE0OFPwr
z<vn(O(ze~liNzKUUzZaQI;+9BH<HLoJ8>v8)!B&{(^xR2d{4e=HP1JaYAaUG<W{yw
zFR^el{En}jMq`y4-&(5g_Rm7j5N02s@FH^E<&gqFf^=cxLhYHqCsYTEl=#hGG|jB>
zZ>DS(Q63ya^E#J=tE=oXfq&RH9C66<3#C2o7l)+eTF1VhYZrDk2hix?+IdDcZ=0|*
zT#pS2*7z#m+Rj|EzD|Wx9r<}LJoJy`5QR0qcWlZMn<4Ji>(<YN+vaJi{Bx|h#YM|h
zTwZQymoKMb8#9Wrxj!P9x(MHgtZJmxk}yL)wEpC&I$`Cw1r+B~ZvNYRx;+n3-q`KN
zKMz^dwH&ajtqEn7XyaDhznT{s5GwD+$W1zho*?*Re10f=X-4Sa_1c3!MlwQo$a0q&
zy@~JHg2tzw2f40fx}A5Od-)CP80oQhc*Gy+KWW#F_8yt$M*qHjbg=paN)t>~r-=z=
z15G*si&h*qN=a_t#5eB{*qF!M+FIMHVaB6%(vzh`oK;6yMi4MZxi-447xz1xD)RIN
zapDT!8=tIKc&u^tdDa6l>IQ!xapecUh^VBTl~>IXE0Sgp6!@7W#L$zRQT%fD(SDS!
zZNoy6Z;yo??9tHEiDM_bqdZ2o?rCebqM}DjF!<-e&=89nQ?VRN`4HAr)A34zm6>1o
zwsXTYo#-VE{`sq(vuTu0HD~H_ocOhtuaXl2KYt3%&YwA7C>UF_IS1gcIt%t8?4G6i
zY{o)@G-oUt-drb}^?hy5pbAB_vs~tYb}hN?>BNcvIfXUVxrd$L2oGr!7R%=!oJF}X
z-%G!W2VsJ}k*A5tKDZ$p^JEp_oRA%ArM5RwDr@)vY^EzH)qJBXwdGF4#VzjSmFd6I
zDRt&yl*^<}o#Jdw7#>@>)nkWiQBYAJ&ptmaIJ;<PYbs4f&}Le!C*Y?=9q|BD4Npj@
z4{rae>{y&Z#>-imD1@%0%M_O>@FFS8!@R+oU)zfB#r$OdM1r^a%$hdE2}a@7Qwtj{
zS+XlBE6-G(XgzlLb?D?++IBMBq7Px4{M-!LSpgICr_UlqPueI2viEggpwk;rW#6w>
zno@)oD-X1n{?)PPCLSg7#s|GWGA^4NT#S5G!6*XDS&4S7HAhyy`u-5B)^Ppv!~|V2
zA4Ilb?@bIgcy(Vu4+-Y+Q5t{2#wi+Nc!EWV4!*h_eXjajVX3&$Aa&n#?J1e7ph8d;
zBCC#2X0l7gP+NfukPd2mlutZk%F|S}<eJ;(Gzd911C(O2?;#%1^#0DyJXQajhr`8P
z-a()R`kTD8ba$dBFDuCh_K}yGOyWxI_j<%5HNNiY+sBl&A>QIUy(4e<oGB*eNJ<d2
z==d61Qq0>+0M57YtINocGL*UwB-wnjw?L=(?%NYUV}4~frWuxbbQjo<Xy4q24HgP9
zi6_d2CUNzsp2!aMN$T&yRb=imt)aKB|IhTUXLByBLD%N6H=;%ob9MX-BDS%tV}=_m
zq4ys=$i4M>H!Y*kYaPONOypU;$*6zAo|1QjEKBdUwA8YnbOCBUP13S?+eISfuTjaj
z0FXIw=ui@5BJ)p=_tn(Z72j~bc{A?Bi4zBynKOZe<Ex%AEsX4C7Ek&0%Mvm7&TZSa
zrMfOo$|)*JPDy5Hrk_3}pSS}nUtUwa(e6HF)AGy4gVxjevrwmJ<@|{?s*NhPEjJb?
z`*O{iPawU)rgS+8-LAjHH9a<VFp{jIqCx;kWxmTaW^7=Ck_W7nu<$GO_3P6(Ri}k4
zX969f`ueuEZ`KR!!_FFc-@kwVSHSf2G@ju&g!k72o~n8E>ldVciE$d*(EZLj0#_&|
zp<(k|@R8ML_Ea=0=%qmdk-2<%G+4`^`2n5e-R6~d@6I`nHeJ}YYZqjc!j3G%tnsWe
z1DLIvnORDS^_S<zL6TaTuAplyU`cQ1N^i#-zQ2BYHyxdphezoa>SZl9Oholjebe~B
zWNUBk>TUi6rw?R7&%2S`+jqeB6*=--ILY(aaH{s)D`?gvJ<*-3cIC>IU{Xm*$w5X&
zi>!@_?()dUNI9xA92~M|&z>FW3+S>8>9Q<ctE9pNzdua8Y_z{M$nBUGQ6NE=MxiJ>
zd(BN8(CT0$BFg~)qH;YqD01H>E^={k1w}_&X9f8Ase|6I$@<@3Ki!>g7f+ZURXHg^
zjo~ZlRy~@n>^+rd@9DMA&9tRoiWdaV^^wJviq~$t?Z=0^3WuJF9oV-o&Zz2{c})=K
z?EJj?#f$e@5%tIS7Z;e+y*!RX(8Zz8u`?B9uA17~R(ZaXgbE+JanPrT^Wz*3tNa<0
zM`An;06^sqY-c}x`WoP$J8fsEu+Z4zk4c>!zgpK4_8jqUlVp{ill_$H*t!A%#g+hY
zEc@NNpy$tHdwcZ+t-94q2;>$(8*Mq}YB*IBlO%C*itD*Kkj+-c4L8zYGYVRMRn{_e
z&qc598IxGI1rjA*y*eSWHeCTDQ_C_a$+<HuH%YY<;~l(9IpVJ$o}}QTh$DJCm&pa%
zOr0{cX;`7s_(o*eVs}XI@@_15s;G|xD6c>i?{CW6+b-jL+2?#APyQeMW%@t%KnerW
zTW^pnYvbuDO@ts43!F67c)#Zk=k56@373yC)r-Xg#6HEwuS9h4Yzzr^&~*(fc{#g{
zcL#*!!8XmQy5@(*y1|}F*QiPj2xHNiq=5pw^F7Sg8igx0E}p!x&%s5x(^U_M38RND
zgbRx+e)A`t&*v{EqkyP5@+I{a%ApzB%fy5ot}>)*%Ea){FflJ{TO?wPgX)^rQkfi*
z{Ym$UceX9NSS$>h^|hQ``p&o|1yRCs&g-?@^B+Ttazr3ab55phC%E5@3>TiZhs@B2
zpo&OuT%Zg?wt!4efDS9pvnVc@ecqp#$Dvj#t$;GeQIeEyfO*TT!$F<e_Adjnd~{Db
z6jbVu_8n4AMnl}IqN*7;Qu;NOjgg-X*ZJNup<vhB-MhYXpT8G=1@7NgxL$SuKfi;s
zJ4*jpAdbOajnRLiRny?w@|N8DkRVeuVv<b8Jf-?FzqjlbOr46?@~KR5?n!!PO|u!(
zr87j-bmBN;Y&vD7OmRd)4*vD|id$``pK}_<zDENPv^G?{N0|Xm9d#<GnibtVptn~u
znwcE}Up^ZwLTu$>?>J-KcLK}V6I`6xspekV_THaUOvl;LW<(l{wEwAhA>&;KdveA9
zOP0pa_1qrE-d6}}Y=oY0d6`hPdd0krHe>ih5Z~Yd$ZcMDqr=`I)}pT<M<2H+P&j_C
z6u}{jlZ*frurjpH>c01h=Oot0oYqap&;Cn}#wd?rCGv0mbB<P;_iZtB&AU}YZ>tJp
zC^zQcb7`aB_IhjC*Tk0H5%3aQ5v^uwZUV58kHJI--~EU|IG|@<gYce}v-Hv%6FPY1
zkfXOnGQn=Kt$xp3-TE?-Q}hnWuUJBXK<e1LQu~|p(&TSJOI5R{qRCZ?siiK<Ira5N
z<+~@P>g6A+Iozd&#T@(-;S~);OVN&N7qe%@<vHzbt}80y`qJ3*A&M$r9Su|_@pg2_
zNqJRAfhj*Xaf$i538wjx@e?o#y#r_z20{^W+oX;*$)_)*^qQm?4-wG5kBZoV7E8+u
zmO>;MQZABROZ(g}5*=h6UK%W^nQmJGXqGRECVD!8E~*6{5p2czqO10j+T>`$LyL;A
z*=?kqaJjqz^v|D#>jY!60VM2PJm`7=1C9%zl}fn+WAtJjyB((y;Qlg+Wg#kvweMh$
zUh59Wkg~A6-LaW7whq)9XB|oEVb_x60zg8K$@BPde;cW<C=bYS3dAWcNl61>Luu-p
zIzVnCWqq}CUKzeNwyDN;D|?S7x>d!R&Ynxz0Fgp{F7Ky!iPjh!bVZ3ewgA2SRcl}{
zVk5!HiGzGZ>dH=bb9X0y_vK3i!qffM$Op7}>#XsCva){94foY|U6z?)KdH)bFq0C>
zgk-&bk&@6Qe9!CBp8=26iUu)wGqKwXu3_`Dv#M9GUJb@eEe$@9tDP!c&6HgJu^+KF
zsG{U#2}jgx^mBlC!m(RUfcLDz_~M8h(c8fSX`>-A0m*=?m0Ka#TqnYmli$2KlZ$Vc
zJ9))V*~G*oIFE;yS5sR%j6o2uZtkwCN`!eNrKFtlXAqPF5re)rpdBCl`Bv&YhH1?Y
zl?E1CHPGplNokSY#K|?!&FTv|!l!rS@A^l%0#!kP;yr!(v}I<f3NnBIlLSG%C(lMp
zy8Lr8CuJtU)2uyB1FZ%u4jT?=bedLHG{7iGf*um<^Q{1-N`bo%kGV`rdHu68AT4=(
zNBMHF*4O#&aQE<-pyL(KyM4Br^-qiBnrjZg7Rar<_4d@MQ(rO$%JffQ6cq!5Xuv@8
zHQZUFA9k=5-<YTi;|EB0nvIPu)uJ<VBtr~TaGgBs4EW0wr5MSP*GLD^$yhLkv_Cy<
zn@8EUWl;3pocz4mtu#P&pyKwgb19UT#JiQ8AK$Tf5p#&Mz&x{nVB<S8qt2Z>$9rAa
zb-}Ve^1PZw$LoR{OMwahjXdBbFV}4kEV4TdzJ@fLg%%oT$L6T!n-Chk9^}^vdX;Hi
zzn-$by2vdcFkdVPT1^XN%v?x=j^iB@uU6}7YE-SQC$}x`!ZMRye@qH9oa$Jgvgt^D
zrRunwrj0%TAvWuV|CK~gZLk45lmxJ?9n^>95FTAFF)>{z&2XJQ9W-y@<@Wsv2dXF_
zl>wHlz5E_^dy)bpdAzgnxY1TGk`u%N?~?2bHW@v9xWX*ihmuFy)p#P7fCgdDe*URe
z{(sX8HtOFYFbS%eW23~PdUs<Gr<wqKqabDAd)`|ikUcv)i{LGwT>Q%X*tMB0Df>Hq
zK>`lQ<I2iM2{_!*S*x%6(UHAa*_~>-dm=QvC$<e*f<zgfdnd1deI}Nbvry)_fzsnv
z4gYbJgk7d76DLd|=+Qg>PzQNip}pgz3y@homegcxD?S#!(ERQl(mW@yEz(SH?yTBL
z%m7^u3gZ6KsGGe*c0A<S2lTE2;kp-1c8J#KBR*$yyV;vK)m{x+0pa7CJPZ)g2G<~i
z48GbMb-D#YbGOHQXIA1`4{e4(oJ~-%1jMvW8fbgL&{7Nt7)~oLSC4kwWjy%!*}_zJ
zevlN(Wgsq>KMD%cFk7J)BY&v8hHDYzVZ>Wse0}i&BIKABB6;+0H$pG^Q;x|LWz*a~
zhl{tdjJKPy8U{_(OCB*e`Zp)YDbW-G{m+R9@xP|xJ0VB@PQ+fmXa@VmAQ7j=Of~sP
zvAkd<o3&p@SyVd~5%NJqhX)fDh7Xv{6Aw((b3!JTAlU0lbSGzsOF*%g*2@<-xz~Q3
zg}HqS6ic6mAdQsGJY1P#8UcBjMw$r`JPy^{Je}DwR-Lm!@x+9Fs|e88&*B+wfcT7&
z{)Esj@}#fp5SqlQCnwPdM<})ly3*H#wC61;CJ@+d&s3ZV%3pq*<WYL5W0Nck!a8?e
z5xV1Gfo%#KBFeCUAmVq9=my}3)_(XjE$>e$FUkT?{0jv{QGVe)OMT}{AelkBr<Lde
z*%$tiho${t4$?gTk%xN|fscQ@?*@Ea<gYwzyB`LTA`=3+o?I>D9vsbsdph;U>#hey
zX%{4t+^9Cyn>M1AwhKcxt43Jz)8Ps87PUO5Cc{h{&cG*}hFGpIBtXHsGkZ!lT92cW
zNDbGXT(i++$Y2MU`pY4F*{Y#uTv+~W3<6Nbiiznn)>O~rKT7T@X#oU8U(_U$9t2qh
z=W;;cZf`E;pPiX+;?Hw#iplc5QA6)l0n>~P@3QnhM7gWXm)V<fCnD}~C0PInji@jk
zTB>7D;12C12bEN2!}l5klaj}Ag(Vg#CV-NuN@1~Wnb8>7rC3BkV@@L7-PD)`$j=nz
z@`WN{7K%Rb@3p@COCJ8dSzr5|hs)2yAO?3wp3cFq$?B5V`wanpKDEjm+04UISy&{!
zXB;+o&PoL8c%ez8%l9-&`<V&WH%w_u3%9A2Vv)=MCU{kVl@;6Y3z$wAbnQvC%3cYV
zI-xjmpZaJY1E!9Ie%=pH?*EC@K98C&!L0CaA0^TLN+3wCo!>Sny5zeJ)0kJQ#Q7^+
zYpuSRPFlW8TD}nFsHH~(9~Kr`7hmXn_t)3Ar2fRDzm)@Mtg}Ii&9xaYC@n1|wIuqM
zu8`8@8X3LgpfU~7Slz?J<7q%ZCxoK35@#$k3&@aGX;-`}y>%Nr1+vlT+}V<yo|v9y
z?v8TBC{gojB3pZjt|O@pR~5jLC*`p=Q#IF-X;@|e5P>lLVMlOycp9V+^QPDf-KFkD
zB_<uOwZp}nGy4hSJXXE=DijI@)kNjw<TfA3U8VZ!+mv-%uMPk0!>S*1wah<{j<+Xi
zy8~FjLhC_^QHc_B8}b*Phq84V@Jhn!L|%J}N<4S}NL#8J(iNbHuaW8jC03xDE%b(N
zHw~SE335V_)ux0TvZn%>Hkduv>=a&}xj+m8MSgJb29Hk8RgihpTN;6nLP=hAlRXy<
z&?|ScPMnXMVX<n5<g>hb7hl<@?`Z1q64XT7M|U?e@9}$WELBP6SoH`E|NNN-oOGqp
zlcL76+V@|o6eAf5ii(<UZf=Q=>nmh6FdwjvLH(qrp&?U!|A7Mu;J)awX!!gO!$FE`
z4Ywu1ZrHkSgJKfJ#ifz_M%sPG%;qDDmuUPI$*q4LIg)Bqwyrt%v%Xc|xi4`qljxV)
z^w|J894>X1Qs_LD3HB3CE-pEEFP%_*Xe)BIu^K4Vr=l_)a5@e-@WY1>!5!On?BEs?
z>!GaKj5lc6^YWY2HNkJwdWEr0`_of?%=+PJF?2B@tp30D#U<N}A2YO30v3t2Cs#^t
zb6c9e$zxcm2SQF8q|L_NUZLMRbZv4soI5l1+rh7aq>T6g$rg|@t}{(&Rwv*B1)%!=
zS7qXE#=09OXfcs_RHD=}efDYXfg1}gzKag+R{9s2`VL|0MGU*VXbzoj0;xR{$y#6E
zUIIFw(XlT4A2LPd`53xgyHpev6%!rLpFh8Q*RHox8$E?hO-*uwNUB&b_-$L*U9{tW
z%9mKmk#-N-8&DM^!?pqy!9}Ok%_Y}I3I`~m4L~yYqr8Ff=6D~FEOgELNbTwEU}(UV
zDXRE9e5juGmxfWY-OE+vPTKWL{fh5>%22K~_ME~kQ7b2FtR5+U`gAv_4GKIu>I3U<
z*{{8f6`Rbz1Z5pjr(Z9Vu7#aYQc_}LWxYzg^6Y5p&-w@p@P-f*-WpYrycw;Vkm-hs
zdi$xfXH}#&*0TFuMwoEK>f0^(c0==PbB*b6=cj&tZP|^I>41M}83a<gSGEuKYEFFV
z*kAj;MpZRa+K{iF`DjLI)J0&xSX(}x&MRFjD|1qxi<?^>zSWxgdJXXYfIWlH>g$^i
z9!_UBs~sB>VO`m#N<7-{E;owL`9*uIm_yY`lUwt(@<>C}gy4P-MRgUG7t9_@H;0FZ
zQ?;@TbPDby0`yHU@x&TF>lnlxF`VdlMfPFEz10$I8sGI6-~2q=1M9t%JFufE%9iDL
zIr(S<Ps>ZTR5__|KHN&rMo!B^50%-4*9+XYFcHrFvSr>InDviZt15CEN00gq9<2&O
zKf&c{j<q$5+YBvL>@z4~r_9OKaLuoUijk*i%lf4$;UTmrea2nzfI>fZ^Z~XrR#9wP
zKJE%1u03<;P}sE=DQnA&s_#A3P?*^CS;aQ2oSsODHi*uA1$2I4u{r&c(Y(||+D3;u
zHi!q1K#Jr~JQ%(Z!z9=121RoH^ef@~r!_y^bE{#&ShpPW5@jE{jB40v66d#_3rmo-
zu}+tT68UBgo64&dB#9J$$$ny@sg`AD*t^|=v>>^Iu5a1;33Pq0R+xmX%jd*lg45@{
z?nELRj?|v?#DIw*K8ecBSs06?q58v1z9&#W*3<3HUl@YHQ(kO+s8VPaG9%Ui*|(hS
zh<6;FFiUe#qw^OXYc<<gZF`^3X|Cy&fMYh(j#n*bPRO{YltQPec@-;8IL87&#N~85
zcgXOuTrf+$nrBAKgxk2ZzeIh+<ts{b&koaIp+IoU*+7x-w5e;G=W|NS=}mfb%n%i@
zwPXcQ?j5`XcZiHmOt=;(iR%m<t75~=FolmLy1r^}8mI!A?PURY%ogAC4g_~x+Bv0z
zwY3!q07$byA*#6zf&s=$5A#$zx95^?2TKc$;e@L8qqsw~>jE}+BX4~~f;(pG0tsA^
zqb&d0eyOfB4Y50{xOLHO79A4JyJzPR+)@?>R4EeNR@aJ<itJp&m#WrF8)C5!t}Hx8
zE0wknJ>txsOH&lvzhel#<BIw^TM?B{v4?paodgWQe!3?uw@B$Kak~R;dtFiCsq#?R
zFDbiV2z$wMacZd0^=u^cOVuWoPx;nq0H_peI|hKy2TA(mb&ry@`rlmsdlr)P&z?qH
zUOGw^+p0VcRwIUuX3dPARbN(f<!djaH`M6ch37bSVR&j!E2_3M+8uWUp(+SfTelL4
z`QE4~!F*jB^sWv825Eb`J0fm*z<4xQ(AAhfL8zW&L75hjd9-zNnZ#g>f|HjoJYilv
zqabDcLykCEQ1~dC@hgt(wzPjM_N?5{5ECT*lLu%MLM*oPpb(KS2%DNg<t9QXl8C<_
z@jh_yXe{y>{gsZb1K_jjhQcMXf?0{*l_Lh(T5WoOO8b!q2s;LYRL^#F_1v@}ssD=!
zXbZLmN}zP^sjN3+!|kqQlNJx0YE_eTJ<Go2FYL&ngH6jaKeHZGMRb!r5I0omFJ@B@
zX0B6#)eap-U_D$2Uj^gA>yg#PDMOl*7%AGw#Z=%|6z#(FBIq_bYb+AGJOSP7k@gHi
zNF^`y*rWr;qkpaYf3YzLY1veGF)ledKdbc2(Qx@}y!(vn*k{*5(tF-GfczL}(fctP
z8XCb{pLQZIStA7HZ5E73VmJ>WnoMdJ<Ho}4O&&pZaLw`(JC**%yrZU1XS(77PPG%{
zQYf0HGjxKMiFx_jg>fRorUlFgpjX)sD|#qt->mSi9w?i2Q2(D4i{9oJYM&X_G~Y?A
zN!Yr}^Q^kkePqG%cZP~kzkK;JB`D~CRe$j^F|RMjJP<h-9?{Ti>I{^5dIBEONBIIW
z8+pD}X&kb)2H+8KiAMMIr$_ir%)sstD!4s1^ZGxTpXOTz&B)`Q!W=N$fA}Ph%RW1*
zl4sqQLm=Z<b3KLo@5bM{b!)!cB1_+4wQPOD1`048@2z@xuR9NxQ{`Is>40va5+mu}
zply_^W1OoqQ47{cu+7#qHfr6taie|5w)|rjE{~wrraQIaaFuvqf4}sccU79qC6K{w
zdEmzvE?lr)r%cS7VqhzH1qtXBFK<I>=Brm12Rt`Q;5~Dsw-fsLon&NW<gQ(7vu~7e
zZjNaMw}_y5%jF|Sj*QTHnPi92JuT;9`di{5EP3jTJwb`bAuRnWhLsJ?Q@sJnL{3R5
zD`yhify%2369^qd3$Ni?-Wk{4&ecI4TZ8$L+s4LVGSP~Qb0h3qD&fmx>QsxdRJ#Xk
zmxS1Eb76rysy<_SRLOK+U?L{<7T6~`0`SH6=D&4c|MKEwd$Egs{mZlQs(l|H(LfRA
zn4ooUe#^?oj~4;(C^!$e=YeA;2LvPqFwk0c=ctr<tmB=ey?jq`rZ>$QPPUZ^-<Bg5
z$_TW9u=+?4iR73U_b|+W>l{D-@HZ6~E_`FO#L7$+3%Oo+AW~y$Y%CQpgZa$&&!`9i
zh<IgTB7Lking27*)&De;m3g$Rvk1TMSkmp@Oq416BPo}THTd~nRiStPo26_OEM;+L
z3*+?F5cz2LUYruWv;B|qU~U**Tch(SySaV17RL+*b0{)z+p!`FOi$VoD5-MglP$oB
zHC*rP24auYZ{#ZboiZ^zM)ilSeT-#w3qSY!@oU<Tp!f_`>cjd2Nl(ynq9{9<^i;l9
za5<Z4<t~)M$5w>=$;k9YgW^011lK|Dh1>Lq*&9cm3If)Bx-^Pnr^W~ov|ItQgz&Ay
z-jXZJkC8PENcVl}dX&>UbMbC7{#2Obb{N3^M;&)0=FAH<6PN;7`<gOS4UYVoQAsXX
zWW(|q;Q`y7n6u4LeOg4D6>;u)|CRwta<Djv##kPx6C~jE^Pp@<EPhrD4(U0-@FTr*
z|E=~6<CBQd*uor?WwqsM1k7CuOM9L4h;7O4{0OLdQzTIY95mZ19lKYqv8EHL-;uhf
z`RD?p3v)}De;xeIx?z!0)Ij_GPWGAqs8yt)b5m_@y9emfnt->!Rfb_4prMc+uYEtK
zuy#8g5Ftn)6Q~IOL{nM927!57A@q!^_2L=8h`Gk+k1XFMjE?nmkC#UCfQo}f?uT`_
z3C^`|0Xqce#{0PPjufV;NYt}1C-d^$M|`#X2}F8&$HEj;1aI%|<6y;67d}K>=>W2Q
zVS^ucf@!Mz&QcJwHZjd;nQeAvw&_QY82<8NI$9{icWsY@x}H}i727rN3%IfJ?fss}
ztEqSsNIRDH4BMIixc6c@ipS#%loL3H!~2swDXG4s2$v(&7%%PAQ7@XWcaLcWuqSa<
z-3kV6y%E^*@xG5}26~6!O?~<=mL=gUdKoFtkq(U^sj3{e4#i>Qn62^)BqkIM>6op=
zDyEH8)E>fYjqh?$MK&oN;edRsz(JESJGam_4}`n%ceadu1^`bn;iHLJL$^C&y~q9B
z9!@S1R69&l;D_yowS%r8^5m%elS=j^e8oLAV#3YNhZZ$VQ9nk8#s(X}d`Jsxum$G!
zYKA3=GcQG*A{kBimdVR-xV0A$;-&^PC_M~zR;ZKrU+w0@k^W#vS*|c@PcwH%uQ8<-
z7Wu)^MyYsE9q@B;t;7V^tRlI_J)=-VL_-h+h$vDHhS~WgSzX6MP!>)2>hp_yt*01%
zj=l#MCX}_>UcK8LVW5*MJLQIIhlwuFf6@kMTsA_rL(G-~HcH6tC|)R%s0D|Iao`He
zOyv~{GfL`ESq#I6f>9w~o!_(}=r=cM(9MQ=I;uv{85$1AA@}3#kLumDw87wn1M58q
z5v2^1%#--`XO?|JBRdjJesVT-%G>n0=7Ou>$@C6r6bKci&UcwJDOq3WA$8}D%UDoT
zQ-iyldG%fO?X>x|*V@?v&XamP2F2H3ym&!a=rV23GL)L=Etq@#W~W{z=v*dlCGx*h
z2@|A}KW`oEPl+~Sqy~T!dY<|^2+{jaUNM6*3I6MwZ!+Tw?12z#e+7UjuxbyUQ!w|n
z@;8ddgUKKbu)|7c86^=_o<Bc5C@8W!Kd`R%R^MC)!F6fXV(Kc-sxHdF<=^z!v1iN@
zI@H_uC<EaSCIMDB2qkNf+-Z-BD2qA$dL!jLJ$M1k-&);T%F53XUH|dp$J}J=Kj{C+
ztj3BR?aS2%?~RAkFG$$%6lX<HoTX)#?-ta_@5PjT3O*gMy>;uAs;X*^%iK?{c3Byj
zv697pHQ?T`x>>;5(!qm|3K7uf5eJoMT$+RCbl&hlv{>jrQoPh3Z7%<0E_U=aOF?B&
zi@_1JCztpjV?l;*{`%G^mp~?Q8Afyh@g<KoSCsYwEkxD?&>-F?DaCknm0SxKz=2mX
zk5BjXp+`PGh6@zsyrbgU3C%1CW+O9(je7zL-{w{hv?%sRN0O;A5qh|}?t9DTC;K@L
z(9@e5YN@Ei0h}`cMI#W3GRY&4q;O-<dGht^b+9E~CBl1Q#UYkYOPkv{{Nu+dcJ`me
zt7jwX&qdU0ondIxh)HB>N4_j|owzF$NrEwOL_hrI))wJNmJ)1ER$@s^PUhy}si%xW
zE5#MR!*Mw|IlkUfW1ZzM!I(hj3R?&?gy!yCiwQx18sP4H?eO!D{FlhlrBkW^MhA7c
zIh04o(q&!uQ!q@ieykDOzIAJg{YXP_$MWiGIv5cb$WXUZhiM2JRqnyH00K!K&n!|z
zi5n_5P)=uKl61RHCo%t=wQK6+(q9CAp{M-eqen<6%N3Q(&-^FDHFH6#Kl5MJXDl+Z
zbw_#Hm%6$jpDND*2LLG<WfXE22uYftVL6i9wJbnjB7J=uUt;q8(-S0vRnSQxb0Pp!
zdFV9w!R(bnRC>SZF>_3MeNllFLGA`Fitj;O6#It>_FLNNL%v{RK^j?kIoNR?=qT&y
zWLC#aG*#)oW5r463U^qblD3zp2<DYOxL))OY>N1b{z#P6WmXjr)Kswg``RQo3+IBD
zd+a-Wbc2_7dX>uni45?Ne8m|vrHl(xsNYhD(yoM#uoKExC|KK4{XOljD-|W7w;A3x
z;v^0JVlcrq7>%6u1&z4*UZ4yZ+s0-}C*~J7%z5ZgpyP#Xv4$LO?>N<3DcNdZDSWhj
z8I=_>&AYw;{lb)aU>H*ef`Z0U!F+ZHna}DcBF2V4G~-97hS1hek=!ahxRGEwjJ$>b
z6p{H%X~70DRVaOIQbb$V|G~`-Mq{Yx_Ucf=&PgoK2xauX;K7l4&R*y1+n*6j=tsHT
z3oMrpd*sB3$xxsV0{1nlY_GiEi-Ky{H55=uLU4;`Fp;I93-xP+yc#QS<j_WV*ur-^
zBK2N67qExA)!e-8VAses>D+~D0r^Z|ZZdQCyfJ{FLCKw7wv4)aV^GoA5$w*A2F!$o
zg=d9%s~YGDy+FYnqrrd`#j<xaQNvaJCO_)BK~LE1c%jD1p?@u<6*a}Ub!oCqEXxob
zK>swzPlwdI+`a?L=j-35V*1}Ut)SGK4~-n8NFb`hrL|4z77H~dS2H43EGUmfa_M$$
z;uDc=@QWbB-LDD1Dg@SF+Nly9zz3|r@>QFHMtGyD=mg1+42gFEF00-Mj>$_#EcQXs
zBe6mr3`2Gj-mJeT-c)^CcLYw}f1pB3ge70b9?I9g4KlNYBIi*NC~|i5=&4QpVwGaq
z6f?c%PGBF6-T$4p7JRiZ2R+4q@2!PCldbTJnt$i5rNN{%_F>=7{abG>=2d$f`#AP*
zytUw^!mv!JYb1v_7G4iHt)PK7iDn(6MDqxay<a%zio}`nVVHxtdp7i{xD_ibo7Nl{
z98W|>M@rNsB&K4#K-QDhbgWzbvYf}(WA@^RU|=*excK@bW4pbj5p36eTct&UA)Wba
zhrhBywjCdW4F1-}0$?>;;1gII6a2tGt2Myld)1~6_dF)g0Fdg^X0Z_Z5~_w3)9e0D
z)uGD{sm{m(1sbusn_ICESf)~cQdXxzQSu%IE=ubhV9yCO;YZ_pi?$t{TyH|g!M(YH
zw*;XZyu6<h8!+qsVr`UV3Y{hh9}1y`5p>cN$ZfM^a=R~IX$q>Ff+G&!zgA{=umk9A
z$p&S0!5I1pj7o-j_eMi`l8v+5uI>bGWtBFAe0RVJNM3<w1O`PKNYsZ89m0dFQ0L8!
zd(obhWtt8TRJI5ia9`ASpRMN5DRRmHP1fAdMkwOopMR>JT>t)Ea)1C5S632}3TGFa
zYcIWWR8vP4O-wp!<Xp1$l>p`pB-ctxQi%U+144NsqDx?gMgHuka@-D9wO3Y1&HWFh
zz$gDiK9cI%wQK5fa!-Y9`rWj9q?UgKg0(LF05gamSw)kD@{qfb;T}n}eT4B0Al@O#
zBLz0+#32a*jhuK#D9<{lcmNM%Wuk%iIXI3WsX(g@Hy?_bDT04>KKzf2K8)WK4+R{k
z))WrrKG?DMsPcG6MlQ66kjGj_A_X<mH8j)PQ4b>WWv;EcZQH)xpu4=cX*TGzME0$s
zr}Erw38eu@)`SwND15$N5x)yAcg7cOlBKBgjcX4@iP&AVv$I=pgx-r4*eZfH{o>jg
z9dprdBP3kkn>0kGf}25a{=vx%om|yGc7>o9FYNj;Gogs(*e}MXczHU58lB9#Lco{V
z*x`@znjp=*Big|E8~`}nSLRg+1VW-Co%pmDw31|T#JJ5Cf3tyEt&B)*q-vyVrt0Qd
z)32Y0!afqk%}yB#YDO*mgaR>m!JJ<yGqamp#yBz9+cUZlULy8`S=WwH-;kE}5(OY?
z9#vLW#vp8MX($Cf3NUs(sHr?>Em`{Ri(I-Pr?p@G<v}2-mwC7YSTzJc3A7r7PnND)
zgJ5I^RfupAyH_B=#Cxo-v`=E=rz+0m{2Do}KyfRQtFgo%y3rAN^DU{l@GY2wvGR%Y
z?%3<6b#>QV?yfCf+WL31L0ZH)<EN<p2gL<ce1M)uO+X;1dnV7uP$5P#inAZD43%rd
zuv^>ys$93noyDuhY}i28)po^=<A0b|$|RWfxYQvZnZw}i7f=^R(zAbIT<#{#71`Ba
zHvZ*^v`F+nr`)Y8B=Wbng-OQozBYSc4<DrZL+f=ch{GYt<GxlP4ufmHz4p?rwlACF
zFto89$G$(clnT#kh6jn{T{mE6SNO$}=tIaw(lEwx1^Z-t7QAF~lCM2I3gi<H3JKZ_
zCIenA6nzzk)&iryfPG>^hHnn3{-j_rkCGBaH=i5z%wSs}p)l#ieGL0fOke^t85?Jf
zsQZ=vXm*9d+y~70rHR+r(RW~baI3WXliw+~B>zqx6tN!(&$bpn6p`@}wng}4@Xvji
zoeF66;PZ;;_*L_QwXgg9W&Wwjv}016d>TV0O8T7PX-<iEtQyeA`eSG;&eUcq)^)Rp
z4WopKoQEQI|ME94#I>N!l!vjiwfLJIsaJdxpLlU!#y)BPQ_}srmkg8Em&Pu$MeL1+
zw5}HPyHgZ8Mq1ujZ1UaMy~nYOEm>-%Hmt^@!R<&L)>K$WwqMbQg(IQ2sWD_2h=ZKV
zu%Qy}@zojSTuaEgZQ2wQ4+%g&zIT0`?Z^2Q@xOe`R}#y)z_TnN{3d(5Vc&faz%kO9
zEwK@4(ArYu6!X)gNXh$nu9c>tjo)Lno=smDyzRqOO|(&OP|^)2(t-N?&xS2%1i6Qe
znv@Q#`XYV00DM9tL#aUMvndC9J|YzArE?i`*pK*bp~)uS{Z}&-{Nf7cKwUN0X!ff`
zz)I9Xjwn#PsuAw?yP*YMw%7@*!){<g66&{!V-9v<q=*uMp-kSwlcII#3D^#}uJZ4_
zvKV&MAG^NepS-e$0XromVjukGyV#Z+QP~93XQ)a5U3<Hy$t|yoibx&5MPf{3-<BIp
zDv)x)lQ>=yvsaGzN_VR{xVat)r)|<Juk7Pk2bHYiLDb-2Rgd~vzH+Mq$^?TB9U|OV
z<UXpV&+btkdQj{5XQ%Af%^AQa;h+Kx4jKyeSG4}spaWUo4;#IY3_7F#$bf(M=fF;s
z#T?GY*hnNLhbE^t>12A7i^27FxUa1gnnR&6l=mSvdICyo0uxeVxejVctYh6F{sJ7w
zul!Y6!}1wIBlw}eIq4dEV5L8!^M^v?+K^QD)dfh-8tB}Cb}d5<LjRdP3en=)adC0z
zI0C={VARuk5M@}3EVfN+F*$JSYwYHA3RXEkvid-?2S;eWI@Vk4lHKpIYUMc5O}_64
zJz!d3^k#OOzL1!dga@9)AYk?!bZzUbFhPrupAAvzs2?6UTM&fc74mp8kw^?@u4G#l
z5R`W=ro=m$(x>3uS=Pt-I`J3FPPy2Xus~9{_B-#X97MK0U*A?no8mZR4TthEVIsG)
zL0imWR2fpy7(k6TDAA4|*uOs>J<0p#DKL%)NKx{koO=w16l}2nAL7wT0QFuYmE%9o
zB;Y=CX8LK>k}H(=!e_ho^7NqcUUS-8=wMaJmL9;bpNDZogu87x9ptrVFo$M#hBG(3
zKp)}LiHMAJ)6@sKHz+dlG*0yHbnLtAZ0`20zq=b2!tMXj*eG}tT4=X7tm10l->!xZ
z-OqXLWh^AcB?yY`AK4_g691zhP#{O%JeeTVjsFLz9H2q;#B5mgE#n)Eq2>&ks5>#o
zL;I(DL>zSEwhNSR+Lp5&0~t2LFs$5N#2pjA^W$;lGp2MX7T0!mzBMgpR|ro4J7iBN
z=<{%vLHhd0HGuDkYP|Dwg-WN^Dg{1hSNj$0-zRuYFvMwgv$L{Z28titf%rWu>nCJP
zfX)`~_D`QaMds0Gcb(2I_(`p;Tw++(0?<B|U1VMGubf`k($exPz`Cyx^@xDoPK<jE
zry9U4k>fPcEjO;2Wst!vLAXi;D<*<*sbHlvgNo7u<ZrLSn*TeC2MNTnf>wAG`u01#
zsHlTrJ>g&p?Wy9qXMbn$|5Aq6tb$H^<(^1MOjG6`?E%?<U3K2weiH5S^9XW;f(<Z3
z^YwKV&a3Er3@BR=;4F;b(9mSmd^lVaytk|Y+M8pO3hLaksC|q3wm@$``1`hWg3BGw
z3jpaAI;{7AV+bp+M@HxW#UI+9WZkz98YsQ|`PN&Jd}xbQ)+j5nDVht7gVRW)FI>3C
z3g>I|eu#p#k=IaGj)k)|Wb?lL;YL4w5YC}MmW3@_w%k4!mgYF{yL=-GU?*xV`eNnC
z3Tfr=k(lxHaiiOAp^azu96WFUeJ0GzlP-csO|dL5A0MPhsl-d2JaguaV(A;tiX}Zm
zeU_6$_U?}EetuK4`Qu?_1No-Ewn6W^5mYfZ&$n?<1t$kYi5Fd#P7d6H_h$-93nj1o
z^*ENytt`;x3SHu;Px9ZLit~CKDiBTP5AB6dz54b!q~X!`v4fo%;3jBD_|_zU_<9{Q
z>(`d&+Tk3K<&~8*=ph0mrbZSv+!iWYI$Rq%9-?cl0#1p8hYzQ~hC*)NVEo@bRt!f!
zFYA+*Aow(p`fi`&6cZ~_jmkbMHP%vf7j<9T`+=SMU8hLv*+^&;$}ITFunqJh=($4S
z40^Ean<oGvCP9dVk4sCx(4iUnM>NJ<D$bzuA|82rL!eQbnVA9bYN$XvRLpDY1}+)p
zss57oH#aXKvIQ4S*Y`$`i!4qJl(GBVQ4HlT1v5}odr?-FO(KKod5@N1|Cvfg8}X5G
zTU17czw$=75f~P}EY%)9nt5`kp`k6Kmdknp*OnVRr?|LO9UO9gHpUnxuKf4szN8%>
zO0G>`f<s6Q;5DfLQF#>~kF;38^RJS;?F?Y~kh+)Gdk4ABxUIV0$q4s31fur=@mw=)
zye&2G8H<#)=i1UxfM7F41{To+eJgV7!WJEMlt_Liw1y&Zj*6U-7=$3ObSJ=>E?Q~!
zAx8PoDC`f4E2EHQ6WDV!G&PZ(HZ3J(w7BQ>{~I?N-srj$EZc9kIa)MTiVRy2+uC8(
z!SbD>st-7wo2epEKD1-rw&BkCp+JxI?N0_4J(i}w$u)}A&IIz>_|=B;WeJo*Ltbrb
z>)h+kIR2v*u*1xL)`h`2Grk^_Wz#;lxq8$Z=k)8VT?0XjuwHPQI#-T3GnO1<@CM0u
zS5P~D2BWA$N;53x>MZIu_hXjmF4a>}xe0<HXt{MM9Ugod8kNqxaH@+yl0t-uVKe~$
zeN0UH?*1BqX?{~Mety;?&>b#m<DbneRgqt8v;<FJU=OSoaTtB>9+k~}sp5<u6jKiq
z{lJ7Bm1hF2Wt9-;xH883ONwUl%GT?lM-4vyE6}+$(<0B;71LN-%U@@b{nlZuCEUFD
zY1+9`DK0)fVTcTVpMB|jI{Tn-Sqa84i`WWowcr`V%C9IZH@U<({~R(s9O}Hs(4DMt
z^4PI*hz~hg_8mrMhrOF&3lG6Rlo+wkTwAd}7)j-DvAVh%3gcNh;41V$td=!6Bm@sf
z76s$Q#l;V>u-IhTu?a;eK*t}xEkz}f424=$(S!qe5<%KO#n0b*fyp5H<;%nH?U3N>
zZH5N@+lGdn&th14^zzZy^7QfJl$?I;fSX|ZV~rw<ExOXXGBJzaRfa?vj0Xu6oj&Bl
zQUxm+02l#wms5uj4u=CS9QNwfD|Choxc@XQvO?q2hPY3kR=<4tA?%e#(Vic5RZ4}$
zM)|hiE`kJd4T`?O9b!FaU#FxfgLn^Z1a0xM0rnmFv*V2#e_&C{PEPNkJ(4?vX7ApR
z&$pU^Z#&HXz-g$fM`r&F=nN~kk97SvZ<H^2?|GYr-VA~p!Q5ITDC@St$IuRX3q)Lz
zH?3*AXldJZZAucAB$qD($9)0=as=Zf7aYe#B=ycf5#lAB)MFtr-*gGaG=4Ux%My5N
zWM&zK;5ga4T<kdF%0zMU^H?=B6?>+^G{7S9m*g<-4LXD8IyAVofKZMaK=6Hqj)mk7
zT{zoFWq*biPNRw*&P`bCcU6jUUkIbDPkGjwWT(QXTvJm+{89XSN#Fzp@FT9HuY##!
ze%=95VTkdb8><6k*qg{pIr7@Ey)%Y;=8glkt&M!h>1OaIq7F=r!Yh;i&S^OM7@9r7
z_6S2+7=})LJbb4F(8S3blz|QV8>WIrQ1k}$cyBu`gSvT1Hgg3sU%;N(Tg3b}NxxwU
zDGZ-JU4|YCjg{C>Z*Ern+UQG^rw0(K0mm$8f*}J;`f3^OS{fQvPz?!?GYv(Zpy0n8
zoyZ&LfKCRB!g}sI=c{btPzvO+%`8~}01`EJ_==1SCl8Op^78Tugg8e~n9O0}MriM)
zXrq08{P^*j>gv=SUg)!g)4&i^26+RHVRC5*fByV=w<Y;0D?2+0ov{U{898*MQl!v4
z;2k9U`T6OPG70*w6S)ih<25r7iw^Xoeg~<I<;K~WnSvV`5PHrURgMCRZG!JI8TO6d
zn+3P5ih*a%HXGyDs%BIx7b9SISwL`FfitbzaxFS(I^IJo4IH#!_w{UX<ji_yPmk`~
zw{I8vqh^+Hp~98W#|Y<lXpXUHJ`?wrl$1mnG_!OC4_<~#aO77wR07Xi=+BTxz(A*%
z0>?gqm}Zse2*$(r8x|c^RkD!T91|HL>XYDM(TjxLrr8c-suI)X)H;O@=83w8B9)-c
zcVTKE3mly~qnRt<?d@{SqN$n^2Hk11O|}2-QI|cm6-_VC8qLG5A@$glY23d3Lng1_
z>-d>-=Tu=)gYi(u5VG#QK3+9*r;=lRg|h~n4i<o{;6SY&nQ_$784;m7UNwEEYS9&-
zK6pOz7ALR4Hhp)NSNHujkhNehPsvy?IqqrvY?57#&yAitzZt82JhpyTfU`Odr>3S}
z!Q<18SKozWI{ez&)Q6_ORe0Y#T&bocVZ657==r{rMZ~r|1)70oKJ8aJT;w!y4$`U3
zT>^mtN-{Z}@M;-Mvb(P5n7w#wKcYDF&4)z@5W$Vxrlw-+OOi{2)D^Y0wJ)VSOYPu{
zsnwHwe2oTku#FfY4pqXJ?;qo_QvEm}Ksa2${8prh-N&)2MN)be16VcB0G2RKNKR&i
zPf8GCrXL7sUo9fXmX1^=2FwV<+_IWWTo>&pTf?4x12457z`c{4oT)8SdUkeiOMZf*
zIeVr}O{pu?b7hQUa~Pn+y|w7%HG&3Q@4URc7pSc_3}|3;>;>u}u%YsU=f`BvQIV#{
zetrTCk>`izha%wex1k{;H?W8Yzvn!4>dc;@&VeVp_Ol;`2){W67}sIIVppIq#-B+1
zGnr)oK6pQn<c@&IDg55D>((Adp}J`@SrF3Y%`CeRlt+Xs2j)rbwTDF=MzzsbCP{8n
ztAz93`~asgz@7Nvh&axQ+vko9xXnSYeCGO!y~fsZo#`H{3zVgN=r-F^u|zgn5$?6E
z(Cz`6aTqr8m2}MvzeRwHuh$_)j#p}E=^HOCEgkMOaLsrN>@J+8?z|pp$vXd=H%$j)
zWmaAZL|Z{9_4<MTh!IXz6NFwVyz`BX5!I?i2#^dAIfTF!>klWy9YH3k83+~8xrPA6
zyzN}d;$2)^yrCpM1814RQELqjdm06WaKQ#zFsMt30~a7Q2S~1bF@qi~w1N(#w(6zX
zT0U+Go9r$>(-`-ToqP8RK`KTSP9re-<eqs|1F#U>#Us5Y=oQ^Od<EWQGhml^^VTHv
z#^Bn`HFB1x;}<Da_$e@<?6OiwJ^nq`MfF#~L!R?{aLZ-!Mb1+~tX&V<rp~vXU8^;u
z+`4@E^5!A|3ig+DUr{G@1si&1)*y4f0>^&9;e%W9xBow(Bmc=-C`vLik5D)TqMhEd
z0unM*lE*za7Co8Tj$rD1rMtFb9;SeGpFVlg4D|&8aNyxuz(SSlIIan9H*I)%dnf<E
zGLHfuqTGN}48lbn(_pv6fdp<oJ@}y;Gz9>raB38$P6IYgR3|K4{uwb0jwnb%60=`U
z5`Ko6A_t^8BV3QH0AGH4+@&3kV=XeAoSe|Bb;m&-Q?^?AV<%ihF%}Brn@0YcnmZ0+
zm~w^)^-j$5jmJ8n&$faVg;HeEmM4P3d@`{52NV}$3AV629?ALn*I|#9p;H4RtP628
z^t^2V-sT~*ZMsl}fXsLe9D#%SHt}*_p$61%16h$1BK*wF`z@`lh@?~rLB9*?J>4*c
zWI}394gluS1|e3i5t`VPz?FS%8+FQlaI}I@?QQYD<A?wGLjFfix#syv3R<0NFW!Wk
OV>lVb%b6E%-2XqC>FDhM
literal 0
HcmV?d00001
diff --git a/tools/upscale_coco/coco.py b/tools/upscale_coco/coco.py
index 11813957e..76cfafd00 100755
--- a/tools/upscale_coco/coco.py
+++ b/tools/upscale_coco/coco.py
@@ -317,7 +317,7 @@ def showAnns(self, anns):
v = kp[2::3]
for sk in sks:
if np.all(v[sk] > 0):
- plt.plot(x[sk], y[sk], linewidth=3, color=c)
+ plt.plot(x[sk], y[sk], linewidth=3, color=c,label=f"keypoint {sk}")
plt.plot(
x[v > 0],
y[v > 0],
@@ -336,6 +336,10 @@ def showAnns(self, anns):
markeredgecolor=c,
markeredgewidth=2,
)
+ plt.xlabel("X Coordinate")
+ plt.ylabel("Y Coordinate")
+ print("Script is running correctly!")
+ plt.show()
p = PatchCollection(
polygons,
facecolor=color,
From 3d868adb6af312093bbf9f59b78a02c6f14ebedc Mon Sep 17 00:00:00 2001
From: "github-actions[bot]" <github-actions[bot]@users.noreply.github.com>
Date: Sun, 19 Jan 2025 01:42:58 +0000
Subject: [PATCH 09/24] [Automated Commit] Format Codebase
---
loadgen/test_settings_internal.cc | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
diff --git a/loadgen/test_settings_internal.cc b/loadgen/test_settings_internal.cc
index 5da0952f8..3f2cd8847 100644
--- a/loadgen/test_settings_internal.cc
+++ b/loadgen/test_settings_internal.cc
@@ -515,11 +515,14 @@ void TestSettingsInternal::LogSummary(AsyncSummary &summary) const {
summary("performance_issue_same : ", performance_issue_same);
summary("performance_issue_same_index : ", performance_issue_same_index);
summary("performance_sample_count : ", performance_sample_count);
- if (sample_concatenate_permutation){
- summary("WARNING: sample_concatenate_permutation was set to true. \n"
- "Generated samples per query might be different as the one in the setting.\n"
- "Check the generated_samples_per_query line in the detailed log for the real\n"
- "samples_per_query value");
+ if (sample_concatenate_permutation) {
+ summary(
+ "WARNING: sample_concatenate_permutation was set to true. \n"
+ "Generated samples per query might be different as the one in the "
+ "setting.\n"
+ "Check the generated_samples_per_query line in the detailed log for "
+ "the real\n"
+ "samples_per_query value");
}
}
From 3060b711082da894b72954db1c5f65f6d2316655 Mon Sep 17 00:00:00 2001
From: sahilavaran <sahil.avaran321@gmail.com>
Date: Thu, 23 Jan 2025 10:19:14 +0000
Subject: [PATCH 10/24] made changes
---
language/bert/README.md | 28 +++++++++++++---------------
1 file changed, 13 insertions(+), 15 deletions(-)
diff --git a/language/bert/README.md b/language/bert/README.md
index fde646755..1a88933a8 100644
--- a/language/bert/README.md
+++ b/language/bert/README.md
@@ -13,15 +13,16 @@ Please see the [new docs site](https://docs.mlcommons.org/inference/benchmarks/l
## Supported Models
-| model | framework | accuracy | dataset | model link | model source | precision | notes |
-| ----- | --------- | -------- | ------- | ---------- | ------------ | --------- | ----- |
-| BERT-Large | TensorFlow | f1_score=90.874% | SQuAD v1.1 validation set | [from zenodo](https://zenodo.org/record/3733868) [from zenodo](https://zenodo.org/record/3939747) | [BERT-Large](https://github.com/google-research/bert), trained with [NVIDIA DeepLearningExamples](https://github.com/NVIDIA/DeepLearningExamples/tree/master/TensorFlow/LanguageModeling/BERT) | fp32 | |
-| BERT-Large | PyTorch | f1_score=90.874% | SQuAD v1.1 validation set | [from zenodo](https://zenodo.org/record/3733896) | [BERT-Large](https://github.com/google-research/bert), trained with [NVIDIA DeepLearningExamples](https://github.com/NVIDIA/DeepLearningExamples/tree/master/TensorFlow/LanguageModeling/BERT), converted with [bert_tf_to_pytorch.py](bert_tf_to_pytorch.py) | fp32 | |
-| BERT-Large | ONNX | f1_score=90.874% | SQuAD v1.1 validation set | [from zenodo](https://zenodo.org/record/3733910) | [BERT-Large](https://github.com/google-research/bert), trained with [NVIDIA DeepLearningExamples](https://github.com/NVIDIA/DeepLearningExamples/tree/master/TensorFlow/LanguageModeling/BERT), converted with [bert_tf_to_pytorch.py](bert_tf_to_pytorch.py) | fp32 | |
-| BERT-Large | ONNX | f1_score=90.067% | SQuAD v1.1 validation set | [from zenodo](https://zenodo.org/record/3750364) | Fine-tuned based on the PyTorch model and converted with [bert_tf_to_pytorch.py](bert_tf_to_pytorch.py) | int8, symetrically per-tensor quantized without bias | See [MLPerf INT8 BERT Finetuning.pdf](MLPerf INT8 BERT Finetuning.pdf) for details about the fine-tuning process |
-| BERT-Large | PyTorch | f1_score=90.633% | SQuAD v1.1 validation set | [from zenodo](https://zenodo.org/record/4792496) | Fine-tuned based on [Huggingface bert-large-uncased pretrained model](https://huggingface.co/bert-large-uncased) | int8, symetrically per-tensor quantized without bias | See README.md at Zenodo link for details about the fine-tuning process |
+| model | framework | accuracy | dataset | model link | model source | precision | notes |
+| ---------- | ---------- | ---------------- | ------------------------- | ------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- |
+| BERT-Large | TensorFlow | f1_score=90.874% | SQuAD v1.1 validation set | [from zenodo](https://zenodo.org/record/3733868) [from zenodo](https://zenodo.org/record/3939747) | [BERT-Large](https://github.com/google-research/bert), trained with [NVIDIA DeepLearningExamples](https://github.com/NVIDIA/DeepLearningExamples/tree/master/TensorFlow/LanguageModeling/BERT) | fp32 | |
+| BERT-Large | PyTorch | f1_score=90.874% | SQuAD v1.1 validation set | [from zenodo](https://zenodo.org/record/3733896) | [BERT-Large](https://github.com/google-research/bert), trained with [NVIDIA DeepLearningExamples](https://github.com/NVIDIA/DeepLearningExamples/tree/master/TensorFlow/LanguageModeling/BERT), converted with [bert_tf_to_pytorch.py](bert_tf_to_pytorch.py) | fp32 | |
+| BERT-Large | ONNX | f1_score=90.874% | SQuAD v1.1 validation set | [from zenodo](https://zenodo.org/record/3733910) | [BERT-Large](https://github.com/google-research/bert), trained with [NVIDIA DeepLearningExamples](https://github.com/NVIDIA/DeepLearningExamples/tree/master/TensorFlow/LanguageModeling/BERT), converted with [bert_tf_to_pytorch.py](bert_tf_to_pytorch.py) | fp32 | |
+| BERT-Large | ONNX | f1_score=90.067% | SQuAD v1.1 validation set | [from zenodo](https://zenodo.org/record/3750364) | Fine-tuned based on the PyTorch model and converted with [bert_tf_to_pytorch.py](bert_tf_to_pytorch.py) | int8, symetrically per-tensor quantized without bias | See [MLPerf INT8 BERT Finetuning.pdf](MLPerf INT8 BERT Finetuning.pdf) for details about the fine-tuning process |
+| BERT-Large | PyTorch | f1_score=90.633% | SQuAD v1.1 validation set | [from zenodo](https://zenodo.org/record/4792496) | Fine-tuned based on [Huggingface bert-large-uncased pretrained model](https://huggingface.co/bert-large-uncased) | int8, symetrically per-tensor quantized without bias | See README.md at Zenodo link for details about the fine-tuning process |
## Disclaimer
+
This benchmark app is a reference implementation that is not meant to be the fastest implementation possible.
## Commands
@@ -45,7 +46,7 @@ Please run the following commands:
- The script [tf_freeze_bert.py] freezes the TensorFlow model into pb file.
- The script [bert_tf_to_pytorch.py] converts the TensorFlow model into the PyTorch `BertForQuestionAnswering` module in [HuggingFace Transformers](https://github.com/huggingface/transformers) and also exports the model to [ONNX](https://github.com/onnx/onnx) format.
-## Loadgen over the Network
+## Loadgen over the Network
```
pip install cm4mlops
@@ -55,22 +56,19 @@ The below CM command will launch the SUT server
```
cm run script --tags=generate-run-cmds,inference --model=bert-99 --backend=pytorch \
---mode=performance --device=cuda --quiet --test_query_count=1000 --network=sut \
---outdirname=results/bert-99-performance
+--mode=performance --device=cuda --quiet --test_query_count=1000 --network=sut
```
-Once the SUT server is launched, the below command can be run on the loadgen node to do issue queries to the SUT nodes. In this command `-sut_servers` has just the localhost address - it can be changed to a comma-separated list of any hostname/IP in the network.
-
+Once the SUT server is launched, the below command can be run on the loadgen node to do issue queries to the SUT nodes. In this command `-sut_servers` has just the localhost address - it can be changed to a comma-separated list of any hostname/IP in the network.
```
cm run script --tags=generate-run-cmds,inference --model=bert-99 --backend=pytorch --rerun \
--mode=performance --device=cuda --quiet --test_query_count=1000 \
---sut_servers,=http://localhost:8000 --network=lon \
---outdirname=results/bert-99-performance-lon
+--sut_servers,=http://localhost:8000 --network=lon
```
If you are not using CM, just add `--network=lon` along with your normal run command on the SUT side.
-On the loadgen node, add `--network=lon` option and `--sut_server <IP1> <IP2>` to the normal command to connect to SUT nodes at IP addresses IP1, IP2 etc.
+On the loadgen node, add `--network=lon` option and `--sut_server <IP1> <IP2>` to the normal command to connect to SUT nodes at IP addresses IP1, IP2 etc.
Loadgen over the network works for `onnxruntime` and `pytorch` backends.
From 9cb4e92f48eb04706e0d393e4f2fa3466e9c8634 Mon Sep 17 00:00:00 2001
From: sahilavaran <sahil.avaran321@gmail.com>
Date: Thu, 23 Jan 2025 10:30:26 +0000
Subject: [PATCH 11/24] made changes in the coco.py
---
tools/upscale_coco/coco.py | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git a/tools/upscale_coco/coco.py b/tools/upscale_coco/coco.py
index 76cfafd00..5bd28ee23 100755
--- a/tools/upscale_coco/coco.py
+++ b/tools/upscale_coco/coco.py
@@ -336,10 +336,7 @@ def showAnns(self, anns):
markeredgecolor=c,
markeredgewidth=2,
)
- plt.xlabel("X Coordinate")
- plt.ylabel("Y Coordinate")
- print("Script is running correctly!")
- plt.show()
+
p = PatchCollection(
polygons,
facecolor=color,
From 9076e4f2b4156f2ab840ab5ae2982deba39bf8e8 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]" <github-actions[bot]@users.noreply.github.com>
Date: Thu, 23 Jan 2025 10:31:11 +0000
Subject: [PATCH 12/24] [Automated Commit] Format Codebase
---
tools/upscale_coco/coco.py | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/tools/upscale_coco/coco.py b/tools/upscale_coco/coco.py
index 5bd28ee23..e6a739f35 100755
--- a/tools/upscale_coco/coco.py
+++ b/tools/upscale_coco/coco.py
@@ -317,7 +317,12 @@ def showAnns(self, anns):
v = kp[2::3]
for sk in sks:
if np.all(v[sk] > 0):
- plt.plot(x[sk], y[sk], linewidth=3, color=c,label=f"keypoint {sk}")
+ plt.plot(
+ x[sk],
+ y[sk],
+ linewidth=3,
+ color=c,
+ label=f"keypoint {sk}")
plt.plot(
x[v > 0],
y[v > 0],
@@ -336,7 +341,7 @@ def showAnns(self, anns):
markeredgecolor=c,
markeredgewidth=2,
)
-
+
p = PatchCollection(
polygons,
facecolor=color,
From ed02ab7ed04023f0cdb258a29a5c2d9e2b29894b Mon Sep 17 00:00:00 2001
From: sahilavaran <139779393+sahilavaran@users.noreply.github.com>
Date: Thu, 23 Jan 2025 10:47:23 +0000
Subject: [PATCH 13/24] Update coco.py
---
tools/upscale_coco/coco.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/tools/upscale_coco/coco.py b/tools/upscale_coco/coco.py
index e6a739f35..62b8f59a9 100755
--- a/tools/upscale_coco/coco.py
+++ b/tools/upscale_coco/coco.py
@@ -321,8 +321,7 @@ def showAnns(self, anns):
x[sk],
y[sk],
linewidth=3,
- color=c,
- label=f"keypoint {sk}")
+ color=c)
plt.plot(
x[v > 0],
y[v > 0],
From c91f2e65b2ae42fe5a6ed38d7f2c9f486ed0d70f Mon Sep 17 00:00:00 2001
From: sahilavaran <139779393+sahilavaran@users.noreply.github.com>
Date: Thu, 23 Jan 2025 10:53:02 +0000
Subject: [PATCH 14/24] Update README.md
---
language/bert/README.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/language/bert/README.md b/language/bert/README.md
index 1a88933a8..85bda14a4 100644
--- a/language/bert/README.md
+++ b/language/bert/README.md
@@ -57,6 +57,7 @@ The below CM command will launch the SUT server
```
cm run script --tags=generate-run-cmds,inference --model=bert-99 --backend=pytorch \
--mode=performance --device=cuda --quiet --test_query_count=1000 --network=sut
+
```
Once the SUT server is launched, the below command can be run on the loadgen node to do issue queries to the SUT nodes. In this command `-sut_servers` has just the localhost address - it can be changed to a comma-separated list of any hostname/IP in the network.
From 6bc50d8f7c0ee1c553aabe2d40c9534e7529b620 Mon Sep 17 00:00:00 2001
From: sahilavaran <139779393+sahilavaran@users.noreply.github.com>
Date: Thu, 23 Jan 2025 10:56:14 +0000
Subject: [PATCH 15/24] Update README.md
---
language/bert/README.md | 1 -
1 file changed, 1 deletion(-)
diff --git a/language/bert/README.md b/language/bert/README.md
index 85bda14a4..1a88933a8 100644
--- a/language/bert/README.md
+++ b/language/bert/README.md
@@ -57,7 +57,6 @@ The below CM command will launch the SUT server
```
cm run script --tags=generate-run-cmds,inference --model=bert-99 --backend=pytorch \
--mode=performance --device=cuda --quiet --test_query_count=1000 --network=sut
-
```
Once the SUT server is launched, the below command can be run on the loadgen node to do issue queries to the SUT nodes. In this command `-sut_servers` has just the localhost address - it can be changed to a comma-separated list of any hostname/IP in the network.
From e00758b637b1041c7ca4f64a920143363983b64e Mon Sep 17 00:00:00 2001
From: sahilavaran <sahil.avaran321@gmail.com>
Date: Thu, 23 Jan 2025 11:02:06 +0000
Subject: [PATCH 16/24] changed coco.py
---
tools/upscale_coco/coco.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tools/upscale_coco/coco.py b/tools/upscale_coco/coco.py
index 5bd28ee23..745a053c0 100755
--- a/tools/upscale_coco/coco.py
+++ b/tools/upscale_coco/coco.py
@@ -317,7 +317,7 @@ def showAnns(self, anns):
v = kp[2::3]
for sk in sks:
if np.all(v[sk] > 0):
- plt.plot(x[sk], y[sk], linewidth=3, color=c,label=f"keypoint {sk}")
+ plt.plot(x[sk], y[sk], linewidth=3, color=c)
plt.plot(
x[v > 0],
y[v > 0],
From fdb343ce4ee67e4bcf3c90528ab51b3f1e6fc3e0 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]" <github-actions[bot]@users.noreply.github.com>
Date: Thu, 23 Jan 2025 11:11:41 +0000
Subject: [PATCH 17/24] [Automated Commit] Format Codebase
---
tools/upscale_coco/coco.py | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/tools/upscale_coco/coco.py b/tools/upscale_coco/coco.py
index 3e7132cd2..b40ab489f 100755
--- a/tools/upscale_coco/coco.py
+++ b/tools/upscale_coco/coco.py
@@ -317,15 +317,17 @@ def showAnns(self, anns):
v = kp[2::3]
for sk in sks:
if np.all(v[sk] > 0):
-<<<<<<< HEAD
+
+
+<< << << < HEAD
plt.plot(x[sk], y[sk], linewidth=3, color=c)
-=======
+== == == =
plt.plot(
x[sk],
y[sk],
linewidth=3,
color=c)
->>>>>>> 6bc50d8f7c0ee1c553aabe2d40c9534e7529b620
+>>>>>> > 6bc50d8f7c0ee1c553aabe2d40c9534e7529b620
plt.plot(
x[v > 0],
y[v > 0],
From d8a0cde595830cc05270d8940ecd56a6af2d56a6 Mon Sep 17 00:00:00 2001
From: sahilavaran <139779393+sahilavaran@users.noreply.github.com>
Date: Thu, 23 Jan 2025 11:17:10 +0000
Subject: [PATCH 18/24] Update coco.py
---
tools/upscale_coco/coco.py | 3 ---
1 file changed, 3 deletions(-)
diff --git a/tools/upscale_coco/coco.py b/tools/upscale_coco/coco.py
index b40ab489f..7c024ea82 100755
--- a/tools/upscale_coco/coco.py
+++ b/tools/upscale_coco/coco.py
@@ -317,9 +317,6 @@ def showAnns(self, anns):
v = kp[2::3]
for sk in sks:
if np.all(v[sk] > 0):
-
-
-<< << << < HEAD
plt.plot(x[sk], y[sk], linewidth=3, color=c)
== == == =
plt.plot(
From e2061f7f96bd57f3356a9e931e0c6f5e5d03e2b9 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]" <github-actions[bot]@users.noreply.github.com>
Date: Thu, 23 Jan 2025 11:17:35 +0000
Subject: [PATCH 19/24] [Automated Commit] Format Codebase
---
tools/upscale_coco/coco.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/tools/upscale_coco/coco.py b/tools/upscale_coco/coco.py
index 7c024ea82..98936631d 100755
--- a/tools/upscale_coco/coco.py
+++ b/tools/upscale_coco/coco.py
@@ -318,6 +318,8 @@ def showAnns(self, anns):
for sk in sks:
if np.all(v[sk] > 0):
plt.plot(x[sk], y[sk], linewidth=3, color=c)
+
+
== == == =
plt.plot(
x[sk],
From 4c88038e38fd3260c0142904ed512bcbdc97da28 Mon Sep 17 00:00:00 2001
From: sahilavaran <sahil.avaran321@gmail.com>
Date: Thu, 23 Jan 2025 13:54:44 +0000
Subject: [PATCH 20/24] merge conflict removed
---
tools/upscale_coco/coco.py | 3 ---
1 file changed, 3 deletions(-)
diff --git a/tools/upscale_coco/coco.py b/tools/upscale_coco/coco.py
index 3e7132cd2..e5b68e566 100755
--- a/tools/upscale_coco/coco.py
+++ b/tools/upscale_coco/coco.py
@@ -317,15 +317,12 @@ def showAnns(self, anns):
v = kp[2::3]
for sk in sks:
if np.all(v[sk] > 0):
-<<<<<<< HEAD
plt.plot(x[sk], y[sk], linewidth=3, color=c)
-=======
plt.plot(
x[sk],
y[sk],
linewidth=3,
color=c)
->>>>>>> 6bc50d8f7c0ee1c553aabe2d40c9534e7529b620
plt.plot(
x[v > 0],
y[v > 0],
From 4de7eed7f3b3a2cc3866692297300e220812e645 Mon Sep 17 00:00:00 2001
From: sahilavaran <139779393+sahilavaran@users.noreply.github.com>
Date: Thu, 23 Jan 2025 13:56:05 +0000
Subject: [PATCH 21/24] Update coco.py
---
tools/upscale_coco/coco.py | 3 ---
1 file changed, 3 deletions(-)
diff --git a/tools/upscale_coco/coco.py b/tools/upscale_coco/coco.py
index 98936631d..a2943c686 100755
--- a/tools/upscale_coco/coco.py
+++ b/tools/upscale_coco/coco.py
@@ -318,9 +318,6 @@ def showAnns(self, anns):
for sk in sks:
if np.all(v[sk] > 0):
plt.plot(x[sk], y[sk], linewidth=3, color=c)
-
-
-== == == =
plt.plot(
x[sk],
y[sk],
From 686c8a394a8454629fae7a2924ea8532952b9b2f Mon Sep 17 00:00:00 2001
From: "github-actions[bot]" <github-actions[bot]@users.noreply.github.com>
Date: Thu, 23 Jan 2025 13:56:37 +0000
Subject: [PATCH 22/24] [Automated Commit] Format Codebase
---
tools/upscale_coco/coco.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/tools/upscale_coco/coco.py b/tools/upscale_coco/coco.py
index a2943c686..6a3267354 100755
--- a/tools/upscale_coco/coco.py
+++ b/tools/upscale_coco/coco.py
@@ -323,6 +323,8 @@ def showAnns(self, anns):
y[sk],
linewidth=3,
color=c)
+
+
>>>>>> > 6bc50d8f7c0ee1c553aabe2d40c9534e7529b620
plt.plot(
x[v > 0],
From 225f81fcf6358aec72b5587a3a4bdc249f1e34af Mon Sep 17 00:00:00 2001
From: sahilavaran <sahil.avaran321@gmail.com>
Date: Thu, 23 Jan 2025 18:40:33 +0000
Subject: [PATCH 23/24] fixed
---
tools/upscale_coco/coco.py | 5 -----
1 file changed, 5 deletions(-)
diff --git a/tools/upscale_coco/coco.py b/tools/upscale_coco/coco.py
index e5b68e566..b9cca9d60 100755
--- a/tools/upscale_coco/coco.py
+++ b/tools/upscale_coco/coco.py
@@ -318,11 +318,6 @@ def showAnns(self, anns):
for sk in sks:
if np.all(v[sk] > 0):
plt.plot(x[sk], y[sk], linewidth=3, color=c)
- plt.plot(
- x[sk],
- y[sk],
- linewidth=3,
- color=c)
plt.plot(
x[v > 0],
y[v > 0],
From 3ffb5df2d54d2c8905872f2a21bfa6d1c9bdad3a Mon Sep 17 00:00:00 2001
From: "github-actions[bot]" <github-actions[bot]@users.noreply.github.com>
Date: Thu, 23 Jan 2025 18:46:55 +0000
Subject: [PATCH 24/24] [Automated Commit] Format Codebase
---
tools/upscale_coco/coco.py | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/tools/upscale_coco/coco.py b/tools/upscale_coco/coco.py
index 368b449dd..02ddc61c6 100755
--- a/tools/upscale_coco/coco.py
+++ b/tools/upscale_coco/coco.py
@@ -318,8 +318,10 @@ def showAnns(self, anns):
for sk in sks:
if np.all(v[sk] > 0):
plt.plot(x[sk], y[sk], linewidth=3, color=c)
-<<<<<<< HEAD
-=======
+
+
+<< << << < HEAD
+== == == =
plt.plot(
x[sk],
y[sk],
@@ -328,7 +330,7 @@ def showAnns(self, anns):
>>>>>> > 6bc50d8f7c0ee1c553aabe2d40c9534e7529b620
->>>>>>> 686c8a394a8454629fae7a2924ea8532952b9b2f
+>>>>>> > 686c8a394a8454629fae7a2924ea8532952b9b2f
plt.plot(
x[v > 0],
y[v > 0],