table2.py

# Import
import matplotlib.pyplot as plt
from tueplots import bundles
import tueplots
plt.rcParams.update(bundles.iclr2024())
import numpy as np
import math

# Constant
TABLE_NAME = "table2_fairness"
ECOLOR ='orange'
BAR_COLOR = tueplots.constants.color.rgb.tue_blue
BAR_WIDTH = 0.93

# Data
MODELS = ['Our 70B', 'Our 13B', 'Our 7B', 'Llama-2 13B', 'Llama-2 7B', 'Vietcuna 7B', 'MixSUra 7x8B', 'Gemini Pro', 'GPT-3.5-turbo', 'GPT-4']
EXCLUDE_MODELS = ['Gemini Pro','GPT-3.5-turbo', 'GPT-4']
qa_task = {
    "XQuAD": {
        "EM": {
            "mean": [0.04, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.13, 0.00, 0.00],
            "std": [0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00],
        },
        "F1": {
            "mean": [0.27, 0.13, 0.13, 0.03, 0.04, 0.00, 0.16, 0.31, 0.24, 0.26],
            "std": [0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00],
        },
    },
    "MLQA": {
        "EM": {
            "mean": [0.03, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.09, 0.00, 0.00],
            "std": [0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00],
        },
        "F1": {
            "mean": [0.25, 0.14, 0.15, 0.04, 0.05, 0.00, 0.17, 0.27, 0.23, 0.24],
            "std": [0.00, 0.00, 0.01, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00],
        },
    },
}

sent_task = {
    "VLSP 2016": {
        "AC": {
            "mean": [0.65, 0.59, 0.74, 0.51, 0.45, 0.04, 0.62, 0.67, 0.66, 0.75],
            "std": [0.01, 0.01, 0.02, 0.01, 0.02, 0.01, 0.00, 0.00, 0.01, 0.01],
        },
        "F1": {
            "mean": [0.49, 0.57, 0.39, 0.1, 0.34, 0.04, 0.62, 0.50, 0.60, 0.74],
            "std": [0.01, 0.01, 0.06, 0.06, 0.01, 0.01, 0.00, 0.00, 0.01, 0.01],
        },
        "AR": {
            "exclude": EXCLUDE_MODELS,
            "mean": [0.58, 0.62, 0.83, 0.56, 0.53, 0.49, 0.59, 0.00, 0.00, 0.00],
            "std": [0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.00, 0.00, 0.00, 0.00],
        },
        "ECE": {
            "mean": [0.13, 0.07, 0.21, 0.32, 0.26, 0.71, 0.30, 0.34, 0.33, 0.41],
            "std": [0.01, 0.01, 0.02, 0.02, 0.02, 0.01, 0.00, 0.00, 0.01, 0.00],
        },
        "A@10": {
            "mean": [0.77, 0.83, 0.98, 0.79, 0.50, 0.05, 0.59, 0.59, 0.52, 0.73],
            "std": [0.04, 0.04, 0.02, 0.04, 0.00, 0.02, 0.00, 0.00, 0.05, 0.04],
        },
    },
    "UiT-VSFC": {
        "AC": {
            "mean": [0.76, 0.75, 0.73, 0.63, 0.51, 0.03, 0.74, 0.79, 0.86, 0.85],
            "std": [0.01, 0.01, 0.01, 0.01, 0.01, 0.00, 0.01, 0.00, 0.01, 0.01],
        },
        "F1": {
            "mean": [0.48, 0.46, 0.73, 0.41, 0.55, 0.03, 0.46, 0.50, 0.71, 0.71],
            "std": [0.01, 0.08, 0.01, 0.02, 0.01, 0.00, 0.00, 0.00, 0.01, 0.01],
        },
        "AR": {
            "exclude": EXCLUDE_MODELS,
            "mean": [0.61, 0.83, 0.78, 0.70, 0.68, 0.55, 0.61, 0.00, 0.00, 0.00],
            "std": [0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.00, 0.00, 0.00, 0.00],
        },
        "ECE": {
            "mean": [0.17, 0.11, 0.13, 0.13, 0.22, 0.50, 0.24, 0.46, 0.52, 0.52],
            "std": [0.01, 0.01, 0.01, 0.01, 0.01, 0.00, 0.00, 0.00, 0.01, 0.01],
        },
        "A@10": {
            "mean": [0.66, 0.88, 0.94, 0.89, 0.64, 0.01, 0.66, 0.82, 0.86, 0.87],
            "std": [0.03, 0.02, 0.01, 0.02, 0.03, 0.01, 0.00, 0.00, 0.02, 0.02],
        },
    },
}

tcl_task = {
    "UiT-VSMEC": {
        "AC": {
            "mean": [0.24, 0.31, 0.29, 0.18, 0.25, 0.15, 0.40, 0.48, 0.44, 0.49],
            "std": [0.02, 0.02, 0.02, 0.02, 0.02, 0.01, 0.00, 0.00, 0.02, 0.02],
        },
        "F1": {
            "mean": [0.14, 0.11, 0.11, 0.08, 0.11, 0.05, 0.36, 0.38, 0.42, 0.47],
            "std": [0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.00, 0.00, 0.02, 0.02],
        },
        "AR": {
            "exclude": EXCLUDE_MODELS,
            "mean": [0.58, 0.58, 0.60, 0.55, 0.57, 0.46, 0.72, 0.00, 0.00, 0.00],
            "std": [0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.00, 0.00, 0.00, 0.00],
        },
        "ECE": {
            "mean": [0.26, 0.23, 0.12, 0.45, 0.22, 0.85, 0.53, 0.34, 0.30, 0.35],
            "std": [0.02, 0.02, 0.02, 0.01, 0.02, 0.01, 0.00, 0.00, 0.02, 0.02],
        },
        "A@10": {
            "mean": [0.37, 0.57, 0.41, 0.44, 0.53, 0.16, 0.79, 0.43, 0.36, 0.36],
            "std": [0.06, 0.06, 0.06, 0.06, 0.06, 0.04, 0.00, 0.00, 0.06, 0.06],
        },
    },
    "PhoATIS": {
        "AC": {
            "mean": [0.15, 0.01, 0.00, 0.02, 0.02, 0.04, 0.81, 0.79, 0.68, 0.83],
            "std": [0.01, 0.01, 0.01, 0.01, 0.00, 0.01, 0.00, 0.00, 0.02, 0.01],
        },
        "F1": {
            "mean": [0.22, 0.05, 0.00, 0.01, 0.06, 0.01, 0.58, 0.67, 0.66, 0.76],
            "std": [0.03, 0.02, 0.00, 0.02, 0.01, 0.00, 0.00, 0.00, 0.03, 0.03],
        },
        "AR": {
            "exclude": EXCLUDE_MODELS,
            "mean": [0.31, 0.58, 0.55, 0.57, 0.57, 0.77, 0.96, 0.00, 0.00, 0.00],
            "std": [0.00, 0.00, 0.00, 0.01, 0.01, 0.01, 0.00, 0.00, 0.00, 0.00],
        },
        "ECE": {
            "mean": [0.81, 0.84, 0.30, 0.90, 0.68, 0.21, 0.14, 0.73, 0.62, 0.77],
            "std": [0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.00, 0.00, 0.02, 0.01],
        },
        "A@10": {
            "mean": [0.13, 0.00, 0.01, 0.01, 0.01, 0.07, 0.91, 0.68, 0.67, 0.87],
            "std": [0.04, 0.01, 0.03, 0.01, 0.01, 0.03, 0.00, 0.00, 0.05, 0.04],
        },
    },
}

td_task = {
    "UiT-ViCTSD": {
        "AC": {
            "mean": [0.41, 0.43, 0.42, 0.27, 0.15, 0.08, 0.69, 0.81, 0.60, 0.87],
            "std": [0.02, 0.02, 0.02, 0.01, 0.00, 0.01, 0.00, 0.00, 0.02, 0.01],
        },
        "F1": {
            "mean": [0.26, 0.29, 0.39, 0.18, 0.11, 0.09, 0.38, 0.43, 0.52, 0.69],
            "std": [0.01, 0.07, 0.01, 0.01, 0.01, 0.01, 0.00, 0.00, 0.02, 0.02],
        },
        "AR": {
            "exclude": EXCLUDE_MODELS,
            "mean": [0.75, 0.66, 0.60, 0.67, 0.62, 0.50, 0.00, 0.00, 0.00, 0.00],
            "std": [0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.00, 0.00, 0.00, 0.00],
        },
        "ECE": {
            "mean": [0.53, 0.36, 0.30, 0.53, 0.67, 0.42, 0.29, 0.31, 0.11, 0.37],
            "std": [0.01, 0.02, 0.01, 0.01, 0.01, 0.01, 0.00, 0.00, 0.02, 0.01],
        },
        "A@10": {
            "mean": [0.33, 0.42, 0.66, 0.57, 0.07, 0.06, 0.78, 0.82, 0.63, 0.86],
            "std": [0.05, 0.05, 0.05, 0.05, 0.03, 0.03, 0.00, 0.00, 0.05, 0.03],
        },
    },
    "UiT-ViHSD": {
        "AC": {
            "mean": [0.15, 0.24, 0.16, 0.16, 0.01, 0.62, 0.56, 0.70, 0.61, 0.76],
            "std": [0.00, 0.01, 0.00, 0.00, 0.00, 0.01, 0.01, 0.00, 0.01, 0.01],
        },
        "F1": {
            "mean": [0.40, 0.15, 0.10, 0.10, 0.01, 0.21, 0.31, 0.37, 0.46, 0.56],
            "std": [0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.01, 0.01],
        },
        "AR": {
            "exclude": EXCLUDE_MODELS,
            "mean": [0.64, 0.61, 0.67, 0.62, 0.56, 0.50, 0.68, 0.00, 0.00, 0.00],
            "std": [0.01, 0.01, 0.01, 0.01, 0.01, 0.00, 0.00, 0.00, 0.00, 0.00],
        },
        "ECE": {
            "mean": [0.58, 0.43, 0.33, 0.59, 0.71, 0.29, 0.32, 0.36, 0.29, 0.43],
            "std": [0.00, 0.01, 0.00, 0.00, 0.00, 0.01, 0.00, 0.00, 0.01, 0.01],
        },
        "A@10": {
            "mean": [0.24, 0.21, 0.28, 0.42, 0.01, 0.62, 0.92, 0.69, 0.62, 0.76],
            "std": [0.02, 0.02, 0.02, 0.02, 0.00, 0.02, 0.00, 0.00, 0.02, 0.02],
        },
    },
}

lm_task = {
    "MLQA-MLM": {
        "EM": {
            "mean": [0.01, 0.02, 0.01, 0.01, 0.00, 0.00, 0.00, 0.03, 0.03, 0.06],
            "std": [0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00],
        },
        "CER": {
            "mean": [0.58, 0.40, 0.40, 0.76, 0.79, 1.04, 0.56, 0.29, 0.29, 0.36],
            "std": [0.01, 0.01, 0.01, 0.00, 0.00, 0.00, 0.00, 0.01, 0.01, 0.01],
        },
        "WER": {
            "mean": [0.70, 0.56, 0.55, 0.89, 0.96, 1.06, 0.63, 0.46, 0.46, 0.41],
            "std": [0.01, 0.01, 0.01, 0.00, 0.00, 0.00, 0.00, 0.01, 0.01, 0.01],
        },
        "CED": {
            "mean": [653.57, 518.38, 492.93, 782.03, 761.38, 940.71, 535.76, 398.19, 398.19, 347.82],
            "std": [12.05, 11.19, 11.32, 11.71, 10.65, 12.48, 0.00, 11.01, 11.01, 10.23],
        },
        "WED": {
            "mean": [150.64, 125.24, 117.82, 192.66, 197.18, 208.05, 133.64, 96.42, 96.42, 86.96],
            "std": [2.73, 2.66, 2.72, 2.83, 2.66, 2.81, 0.00, 2.54, 2.54, 2.41],
        },
        "PLX": {
            "exclude": EXCLUDE_MODELS,
            "mean": [1.25, 1.48, 1.22, 1.27, 1.75, 1.40, 1.00, 0.00, 0.00, 0.00],
            "std": [0.06, 0.11, 0.01, 0.04, 0.20, 0.00, 0.00, 0.00, 0.00, 0.00],
        },
    },
    "VSEC": {
        "EM": {
            "mean": [0.30, 0.32, 0.20, 0.15, 0.12, 0.06, 0.07, 0.59, 0.59, 0.67],
            "std": [0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00],
        },
        "CER": {
            "mean": [0.11, 0.07, 0.54, 0.07, 0.35, 4.78, 0.20, 0.06, 0.06, 0.01],
            "std": [0.00, 0.00, 0.01, 0.00, 0.01, 0.06, 0.00, 0.00, 0.00, 0.00],
        },
        "WER": {
            "mean": [0.14, 0.21, 0.67, 0.22, 0.48, 4.80, 0.29, 0.19, 0.19, 0.02],
            "std": [0.00, 0.00, 0.01, 0.00, 0.01, 0.06, 0.00, 0.00, 0.00, 0.00],
        },
        "CED": {
            "mean": [15.19, 2.98, 41.77, 3.39, 47.54, 634.48, 25.96, 1.99, 1.99, 1.30],
            "std": [0.42, 0.11, 1.57, 0.16, 0.85, 8.58, 0.00, 0.08, 0.08, 0.04],
        },
        "WED": {
            "mean": [4.12, 1.24, 10.12, 1.52, 11.82, 145.12, 8.79, 0.74, 0.74, 0.54],
            "std": [0.11, 0.03, 0.35, 0.04, 0.19, 1.94, 0.00, 0.02, 0.02, 0.01],
        },
        "PLX": {
            "exclude": EXCLUDE_MODELS,
            "mean": [1.13, 1.15, 1.07, 1.01, 1.06, 1.46, 1.00, 0.00, 0.00, 0.00],
            "std": [0.00, 0.00, 0.00, 0.00, 0.00, 0.01, 0.00, 0.00, 0.00, 0.00],
        },
    },
}



drawing_tasks = {
    "Question-Answering": qa_task, 
    # "Summarization": sum_task, 
    "Sentiment Analysis": sent_task, 
    "Text Classification": tcl_task, 
    # "Knowledge": kn_task, 
    "Toxic Detection": td_task, 
    "Language Modeling": lm_task, 
    # "Reasoning": reasoning_task,
}
for task_name, task in drawing_tasks.items():
    datasets = task.keys()
    for dataset in datasets:
        metrics = task[dataset].keys()
        for metric in metrics:
            try:
                exclude_flag = task[dataset][metric]["exclude"] if "exclude" in task[dataset][metric].keys() else []
                tmp_model = list(filter(lambda x: x not in exclude_flag, MODELS))
                mean = task[dataset][metric]["mean"][:len(tmp_model)]
                std = task[dataset][metric]["std"][:len(tmp_model)]

            
                plt.figure(figsize=(7, 10))  # Adjust figure size as needed
                # print(avg_std_F1_qa)
                # plt.figure(figsize=(6, 10))  # Adjust figure size as needed
                # Create horizontal bar plot
                y_pos = np.arange(len(tmp_model))#0069aa

                plt.barh(y_pos, list(reversed(mean)), align='center', color=BAR_COLOR, ecolor=ECOLOR, xerr=list((reversed(std))), error_kw=dict(lw=2, capsize=3, capthick=1, color='#fff'),  height=BAR_WIDTH)
                # plt.errorbar(y_pos, accuracies, xerr=std, lw=2, capsize=5, capthick=2, color='#fff')
                plt.yticks(y_pos, reversed(tmp_model), fontsize=15)
                plt.xticks(fontsize=15)
                # plt.xlabel(task_name, fontsize=15)
                plt.title(f"{dataset}\n{metric}", fontsize=15)

                # Add grid and limit y-axis to 1.0
                plt.grid(axis='x', linestyle='--', alpha=0.8)
                plt.xlim(math.floor(min(0, min(mean))), math.ceil(max(mean)))

                plt.tight_layout()  # Ensures labels are not cut off
                plt.savefig(f"{TABLE_NAME}/{TABLE_NAME}_{task_name}_{dataset}_{metric}.pdf")  # Save to a file (optional)
                plt.close()
            except Exception as e:
                print(str(e))
                print(task_name)
                print(dataset)
                print(metric)
                exit(0)
# plt.show()