Merge branch 'master' into dev

language/gpt-j/evaluation.py

@@ -56,6 +56,7 @@ def main():
     dataset_path = args.dataset_file
     metric = evaluate.load("rouge")
     nltk.download("punkt")
+    nltk.download('punkt_tab')
 
     tokenizer = AutoTokenizer.from_pretrained(
         model_name,

main.py

@@ -521,6 +521,7 @@ def get_common_info(spaces, implementation, model):
 
         if implementation.lower() == "reference" and model.lower() not in [
                 "pointpainting"]:
+
             info += f"{pre_space}    - `_r4.1-dev` could also be given instead of `_r5.0-dev` if you want to run the benchmark with the MLPerf version being 4.1.\n\n"
         if model == "rgat":
             info += f"{pre_space}    - Add `--env.MLC_DATASET_IGBH_PATH=<Path to IGBH dataset>` if you have already downloaded the dataset. The path will be automatically mounted when using docker run.\n\n"

tools/submission/submission_checker.py

@@ -735,19 +735,19 @@
 ACC_PATTERN = {
     "acc": r"^(?:\{\"accuracy|accuracy)[\": ]*=?\s*([\d\.]+).*",
     "AUC": r"^AUC=([\d\.]+).*",
-    "mAP": r".*'(?:mAP|Total)':\s*([\d\.]+)",
+    "mAP": r".*(?:mAP=|Total:)\s*([\d.]+)",
     "bleu": r"^BLEU\:\s*([\d\.]+).*",
     "F1": r"^{[\"\']exact_match[\"\']\:\s*[\d\.]+,\s*[\"\']f1[\"\']\:\s*([\d\.]+)}",
     "WER": r"Word Error Rate\:.*, accuracy=([0-9\.]+)%",
     "DICE": r"Accuracy\:\s*mean\s*=\s*([\d\.]+).*",
-    "ROUGE1": r".*'rouge1':\s([\d.]+).*",
-    "ROUGE2": r".*'rouge2':\s([\d.]+).*",
-    "ROUGEL": r".*'rougeL':\s([\d.]+).*",
+    "ROUGE1": r".*'rouge1':\s+'?([\d.]+)'?.*",
+    "ROUGE2": r".*'rouge2':\s+'?([\d.]+)'?.*",
+    "ROUGEL": r".*'rougeL':\s+'?([\d.]+)'?.*",
     "ROUGELSUM": r".*'rougeLsum':\s([\d.]+).*",
     "GEN_LEN": r".*'gen_len':\s([\d.]+).*",
     "TOKENS_PER_SAMPLE": r".*'tokens_per_sample':\s([\d.]+).*",
-    "CLIP_SCORE": r".*'CLIP_SCORE':\s([\d.]+).*",
-    "FID_SCORE": r".*'FID_SCORE':\s([\d.]+).*",
+    "CLIP_SCORE": r".*'CLIP_SCORE':\s.([\d.]+).*",
+    "FID_SCORE": r".*'FID_SCORE':\s.([\d.]+).*",
     "gsm8k_accuracy": r".*'gsm8k':\s([\d.]+).*",
     "mbxp_accuracy": r".*'mbxp':\s([\d.]+).*",
     "exact_match": r".*'exact_match':\s([\d.]+).*"