Report improvement - support output of IDs in a json file

main.py

@@ -28,14 +28,16 @@ def mlperf_inference_implementation_readme(
         content = ""
 
         execution_envs = ["Docker", "Native"]
-        code_version = "r5.0-dev"
+        code_version = "r4.1-dev"
         implementation_run_options = []
 
         if model == "rnnt":
             code_version = "r4.0"
 
         if implementation == "reference":
             # Tip
+            if model != "rnnt":
+                code_version = "r5.0-dev"
             if "99.9" not in model and implementation_tips:
                 content += f"\n{pre_space}!!! tip\n\n"
                 content += f"{pre_space}    - MLCommons reference implementations are only meant to provide a rules compliant reference implementation for the submitters and in most cases are not best performing. If you want to benchmark any system, it is advisable to use the vendor MLPerf implementation for that system like Nvidia, Intel etc.\n\n"
@@ -502,7 +504,8 @@ def get_common_info(spaces, implementation, model):
         info += f"\n{pre_space}!!! tip\n\n"
         info += f"{pre_space}    - Number of threads could be adjusted using `--threads=#`, where `#` is the desired number of threads. This option works only if the implementation in use supports threading.\n\n"
         info += f"{pre_space}    - Batch size could be adjusted using `--batch_size=#`, where `#` is the desired batch size. This option works only if the implementation in use is supporting the given batch size.\n\n"
-        info += f"{pre_space}    - `_r4.1-dev` could also be given instead of `_r5.0-dev` if you want to run the benchmark with the MLPerf version being 4.1.\n\n"
+        if implementation.lower() == "reference":
+            info += f"{pre_space}    - `_r4.1-dev` could also be given instead of `_r5.0-dev` if you want to run the benchmark with the MLPerf version being 4.1.\n\n"
         if model == "rgat":
             info += f"{pre_space}    - Add `--env.CM_DATASET_IGBH_PATH=<Path to IGBH dataset>` if you have already downloaded the dataset. The path will be automatically mounted when using docker run.\n\n"
             info += f"{pre_space}    - Add `--env.CM_ML_MODEL_RGAT_CHECKPOINT_PATH=<Path to R-GAT model checkpoint>` if you have already downloaded the model. The path will be automatically mounted when using docker run.\n\n"

tools/submission/generate_final_report.py

@@ -283,16 +283,34 @@ def MakeUniqueID(x):
             "Unique ID (e.g. for Audit)"],
         inplace=True,
     )
-    id_dict = {
-        key: 1 + value
-        for (value, key) in enumerate(pd.unique(df["Unique ID (e.g. for Audit)"]))
+    if os.path.exists("ids.json"):
+        with open("ids.json", "r") as f:
+            id_dict = json.load(f)
+    else:
+        id_dict = {}
+    cur_keys = id_dict.keys()
+    cur_ids = id_dict.values()
+    new_keys = [
+        a for a in pd.unique(
+            df["Unique ID (e.g. for Audit)"]) if a not in cur_keys]
+    if cur_ids:
+        max_cur_id = max(cur_ids)
+    else:
+        max_cur_id = 0
+    id_dict_new = {
+        key: 1 + value + max_cur_id
+        for (value, key) in enumerate(new_keys)
     }
+    id_dict.update(id_dict_new)
+
     df["ID"] = df.apply(
         lambda x: "{}-{:04}".format(
             args.version, id_dict[x["Unique ID (e.g. for Audit)"]]
         ),
         axis=1,
     )
+    with open("ids.json", "w") as f:
+        f.write(json.dumps(id_dict, indent=4))
 
     for category in ["closed", "open", "network"]:
         for suite in ["datacenter", "edge"]: