Merge pull request #3 from arg-tech/cassi2cass

CASS equation correction and explicit CASSi inclusion
arg-tech · Oct 2, 2024 · a704b91 · a704b91
2 parents 6e35a36 + 6fb307b
commit a704b91
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 15 deletions.
diff --git a/app/evaluation_metrics.py b/app/evaluation_metrics.py
@@ -110,15 +110,20 @@ def text_similarity(data1, data2):
 
     @staticmethod
     def CASS_calculation(text_sim_ss,k_graph):
-            if text_sim_ss == 'Error Text Input Is Empty' or text_sim_ss == 'Error: Source Text Was Different as Segmentations differ in length':
+            if text_sim_ss == 'Error Text Input Is Empty' or text_sim_ss == 'None:Error! Source Text Was Different as Segmentations differ in length':
                 overall_sim='None'
-                return overall_sim
             else:
+                overall_sim = 2((float(k_graph)*float(text_sim_ss))/(float(k_graph)+float(text_sim_ss)))
 
-             # overall_sim = (k_graph + text_sim_ss) / 2
-              overall_sim = (float(k_graph) + float(text_sim_ss)) / 2
+            return overall_sim
 
-              return overall_sim
+    @staticmethod
+    def CASSi_calculation(text_sim_ss,k_graph):
+            if text_sim_ss == 'Error Text Input Is Empty' or text_sim_ss == 'None:Error! Source Text Was Different as Segmentations differ in length':
+                overall_sim='None'
+            else:
+                overall_sim = (float(k_graph) + float(text_sim_ss)) / 2
+            return overall_sim
 
 
     # f1 0-1

diff --git a/app/routes.py b/app/routes.py
@@ -48,15 +48,17 @@ def get_data_file():
                 data1, data2)
             kappa = eval.kappa_calculation(all_s_a_cm, prop_rels_comp_cm, loc_ya_rels_comp_cm, prop_ya_comp_cm, loc_ta_cm, prop_ya_cm)
             Text_similarity = eval.text_similarity(data1, data2)
-            CASS = eval.CASS_calculation(Text_similarity, kappa)
-            F1 = eval.F1_Macro_calculation(all_s_a_cm, prop_rels_comp_cm, loc_ya_rels_comp_cm, prop_ya_comp_cm, loc_ta_cm, prop_ya_cm)
+            cass = eval.CASS_calculation(Text_similarity, kappa)
+            cassi = eval.CASSi_calculation(Text_similarity, kappa)
+            f1_score = eval.F1_Macro_calculation(all_s_a_cm, prop_rels_comp_cm, loc_ya_rels_comp_cm, prop_ya_comp_cm, loc_ta_cm, prop_ya_cm)
             Acc = eval.accuracy_calculation(all_s_a_cm, prop_rels_comp_cm, loc_ya_rels_comp_cm, prop_ya_comp_cm, loc_ta_cm, prop_ya_cm)
             U_Alpha = eval.u_alpha_calculation(all_s_a_cm, prop_rels_comp_cm, loc_ya_rels_comp_cm, prop_ya_comp_cm, loc_ta_cm, prop_ya_cm)
 
             results = {
-                "Macro F1": F1,
+                "Macro F1": f1_score,
                 "Accuracy": Acc,
-                "CASS": CASS,
+                "CASS": cass,
+                "CASSi": cassi,
                 "Text Similarity": Text_similarity,
                 "U-Alpha": U_Alpha,
                 "Kappa": kappa
@@ -93,15 +95,17 @@ def get_data_file():
                     data1, data2)
                 kappa = eval.kappa_calculation(all_s_a_cm, prop_rels_comp_cm, loc_ya_rels_comp_cm, prop_ya_comp_cm, loc_ta_cm, prop_ya_cm)
                 Text_similarity = eval.text_similarity(data1, data2)
-                CASS = eval.CASS_calculation(Text_similarity, kappa)
-                F1 = eval.F1_Macro_calculation(all_s_a_cm, prop_rels_comp_cm, loc_ya_rels_comp_cm, prop_ya_comp_cm, loc_ta_cm, prop_ya_cm)
+                cass = eval.CASS_calculation(Text_similarity, kappa)
+                cassi = eval.CASSi_calculation(Text_similarity, kappa)
+                f1_score = eval.F1_Macro_calculation(all_s_a_cm, prop_rels_comp_cm, loc_ya_rels_comp_cm, prop_ya_comp_cm, loc_ta_cm, prop_ya_cm)
                 Acc = eval.accuracy_calculation(all_s_a_cm, prop_rels_comp_cm, loc_ya_rels_comp_cm, prop_ya_comp_cm, loc_ta_cm, prop_ya_cm)
                 U_Alpha = eval.u_alpha_calculation(all_s_a_cm, prop_rels_comp_cm, loc_ya_rels_comp_cm, prop_ya_comp_cm, loc_ta_cm, prop_ya_cm)
 
                 results = {
-                    "Macro F1": F1,
+                    "Macro F1": f1_score,
                     "Accuracy": Acc,
-                    "CASS": CASS,
+                    "CASS": cass,
+                    "CASSi": cassi,
                     "Text Similarity": Text_similarity,
                     "U-Alpha": U_Alpha,
                     "Kappa": kappa

diff --git a/app/templates/docs.html b/app/templates/docs.html
@@ -2,10 +2,10 @@
 <html>
 <body>
 
-<h1> Evaluation Metrics (F1 Macro/Accuracy/CASS/Text Similarity/Kappa/U-Alpha) Computation</h1>
+<h1> Evaluation Metrics (F1 Macro/Accuracy/CASS/CASSi/Text Similarity/Kappa/U-Alpha) Computation</h1>
 
 <p> <b>Expected Input:</b> Two comprehensive xAIF files containing all information nodes and text segments</p>
 <h3> Functionality: first, this web service considers two xAIF graph match and then based on it, compares two xAIF files to compute evaluation metrics for statistical purposes or annotation agreements . </h3>
-<p> <b>Expected Output: F1 Macro/Accuracy/CASS/Text Similarity/Kappa/U-Alpha</b> </p>
+<p> <b>Expected Output: F1 Macro/Accuracy/CASS/CASSi/Text Similarity/Kappa/U-Alpha</b> </p>
 </body>
 </html>