Feature/new metrics (#322)

* Update model methods to run background task when creating the endpoint and creating score endpoint for runpod to use * Handle error from Runpod, URL inference from yaml file * Update backend/app/api/endpoints/base/score.py Co-authored-by: Rafael Mosquera <[email protected]> * Rename method * rename endpoint * Print background tasks and correct typo * Correct typos and correct schemas * model uid * remove c * New metrics for MLSuperb challenge * Correct pretty name --------- Co-authored-by: Rafael Mosquera <[email protected]>
mlcommons · Jan 22, 2025 · 275c06d · 275c06d
1 parent a7d469a
commit 275c06d
Show file tree

Hide file tree

Showing 4 changed files with 126 additions and 0 deletions.
diff --git a/api/evaluation/metrics/metrics.py b/api/evaluation/metrics/metrics.py
@@ -145,6 +145,60 @@ def get_kullback_leibler_divergence_meta(task=None):
     }
 
 
+def get_standard_accuracy_meta(task=None):
+    return {
+        "unit": "%",
+        "pretty_name": "Accuracy",
+        "utility_direction": 1,
+        "offset": 0,
+    }
+
+
+def get_CER_meta(task=None):
+    return {
+        "unit": "%",
+        "pretty_name": "CER",
+        "utility_direction": -1,
+        "offset": 0,
+    }
+
+
+def get_STD_CER_meta(task=None):
+    return {
+        "unit": "%",
+        "pretty_name": "STD CER",
+        "utility_direction": -1,
+        "offset": 0,
+    }
+
+
+def get_CER_15_WORSE_meta(task=None):
+    return {
+        "unit": "%",
+        "pretty_name": "CER 15 WORSE",
+        "utility_direction": -1,
+        "offset": 0,
+    }
+
+
+def get_dialect_Accuracy_meta(task=None):
+    return {
+        "unit": "%",
+        "pretty_name": "Dialect Accuracy",
+        "utility_direction": 1,
+        "offset": 0,
+    }
+
+
+def get_dialect_CER_meta(task=None):
+    return {
+        "unit": "%",
+        "pretty_name": "Dialect CER",
+        "utility_direction": -1,
+        "offset": 0,
+    }
+
+
 def get_dataperf_fraction_of_fixes(required_fixes, total_fixes):
     fraction_of_fixes = required_fixes / total_fixes
     return fraction_of_fixes

diff --git a/api/evaluation/metrics/metrics_dicts.py b/api/evaluation/metrics/metrics_dicts.py
@@ -55,4 +55,10 @@
     "dataperf_balanced_accuracy": metrics.get_dataperf_balanced_accuracy_meta,
     "chrf": metrics.get_chrf_meta,
     "kullback_leibler_divergence": metrics.get_kullback_leibler_divergence_meta,
+    "Standard_Accuracy": metrics.get_standard_accuracy_meta,
+    "Standard_CER": metrics.get_CER_meta,
+    "Standard_STD_CER": metrics.get_STD_CER_meta,
+    "Standard_CER_15_WORSE": metrics.get_CER_15_WORSE_meta,
+    "Dialect_Accuracy": metrics.get_dialect_Accuracy_meta,
+    "Dialect_CER": metrics.get_dialect_CER_meta,
 }
diff --git a/backend/app/domain/services/builder_and_evaluation/eval_utils/metrics.py b/backend/app/domain/services/builder_and_evaluation/eval_utils/metrics.py
@@ -175,6 +175,60 @@ def get_kullback_leibler_divergence_meta(task=None):
     }
 
 
+def get_standard_accuracy_meta(task=None):
+    return {
+        "unit": "%",
+        "pretty_name": "Accuracy",
+        "utility_direction": 1,
+        "offset": 0,
+    }
+
+
+def get_CER_meta(task=None):
+    return {
+        "unit": "%",
+        "pretty_name": "CER",
+        "utility_direction": -1,
+        "offset": 0,
+    }
+
+
+def get_STD_CER_meta(task=None):
+    return {
+        "unit": "%",
+        "pretty_name": "STD CER",
+        "utility_direction": -1,
+        "offset": 0,
+    }
+
+
+def get_CER_15_WORSE_meta(task=None):
+    return {
+        "unit": "%",
+        "pretty_name": "CER_15_WORSE",
+        "utility_direction": -1,
+        "offset": 0,
+    }
+
+
+def get_dialect_Accuracy_meta(task=None):
+    return {
+        "unit": "%",
+        "pretty_name": "Dialect Accuracy",
+        "utility_direction": 1,
+        "offset": 0,
+    }
+
+
+def get_dialect_CER_meta(task=None):
+    return {
+        "unit": "%",
+        "pretty_name": "Dialect CER",
+        "utility_direction": -1,
+        "offset": 0,
+    }
+
+
 def get_chrf_pp(predictions: list, targets: list):
     """Chrf++ metric.
 

diff --git a/backend/app/domain/services/builder_and_evaluation/eval_utils/metrics_dicts.py b/backend/app/domain/services/builder_and_evaluation/eval_utils/metrics_dicts.py
@@ -11,6 +11,8 @@
     get_accuracy_meta,
     get_bleu,
     get_bleu_meta,
+    get_CER_15_WORSE_meta,
+    get_CER_meta,
     get_chrf,
     get_chrf_meta,
     get_chrf_pp_meta,
@@ -22,6 +24,8 @@
     get_dataperf_f1_meta,
     get_dataperf_fraction_of_fixes,
     get_dataperf_fraction_of_fixes_meta,
+    get_dialect_Accuracy_meta,
+    get_dialect_CER_meta,
     get_examples_per_second,
     get_examples_per_second_meta,
     get_f1,
@@ -41,6 +45,8 @@
     get_sp_bleu_meta,
     get_squad_f1,
     get_squad_f1_meta,
+    get_standard_accuracy_meta,
+    get_STD_CER_meta,
     get_unperturbed_percent,
     get_vqa_accuracy,
     get_vqa_accuracy_meta,
@@ -97,4 +103,10 @@
     "dataperf_balanced_accuracy": get_dataperf_balanced_accuracy_meta,
     "chrf": get_chrf_meta,
     "kullback_leibler_divergence": get_kullback_leibler_divergence_meta,
+    "Standard_Accuracy": get_standard_accuracy_meta,
+    "Standard_CER": get_CER_meta,
+    "Standard_STD_CER": get_STD_CER_meta,
+    "Standard_CER_15_WORSE": get_CER_15_WORSE_meta,
+    "Dialect_Accuracy": get_dialect_Accuracy_meta,
+    "Dialect_CER": get_dialect_CER_meta,
 }