support supergpqa

examples/eval_supergpqa.py

@@ -0,0 +1,14 @@
+from mmengine import read_base
+
+with read_base():
+    # from opencompass.configs.datasets.supergpqa.supergpqa_mixed_gen_d00bdd import \
+    #     supergpqa_mixed_datasets as mixed_datasets
+    from opencompass.configs.datasets.supergpqa.supergpqa_single_0_shot_gen import \
+        supergpqa_0shot_single_datasets as zero_shot_datasets
+    # from opencompass.configs.datasets.supergpqa.supergpqa_single_3_shot_gen import \
+    #     supergpqa_3shot_single_datasets as three_shot_datasets
+    from opencompass.configs.models.hf_internlm.hf_internlm2_5_7b import \
+        models as hf_internlm2_5_7b
+
+datasets = zero_shot_datasets
+models = hf_internlm2_5_7b

opencompass/configs/datasets/supergpqa/supergpqa_single_0_shot_gen.py

@@ -0,0 +1,55 @@
+from opencompass.datasets.supergpqa.supergpqa import SuperGPQADataset, SuperGPQAEvaluator
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+
+
+supergpqa_0shot_single_datasets = []
+prompt_template = dict(
+    type=PromptTemplate,
+    template=dict(
+        begin=[
+            dict(
+                role='HUMAN',
+                prompt=''
+            )
+        ],
+        round=[
+            dict(
+                role='HUMAN',
+                prompt='{infer_prompt}' # f-string
+            )
+        ]
+    )
+)
+
+# Reader configuration
+reader_cfg = dict(
+    input_columns=['infer_prompt'],
+    output_column='answer_letter',
+)
+
+# Inference configuration
+infer_cfg = dict(
+    prompt_template=prompt_template,
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=1024),
+)
+
+# Evaluation configuration
+eval_cfg = dict(
+    evaluator=dict(type=SuperGPQAEvaluator),
+    pred_role='BOT',
+)
+supergpqa_dataset = dict(
+    type=SuperGPQADataset,
+    abbr='supergpqa',
+    path='opencompass/supergpqa',
+    prompt_mode='zero-shot',
+    reader_cfg=reader_cfg,
+    infer_cfg=infer_cfg,
+    eval_cfg=eval_cfg,
+)
+# print(type(supergpqa_0shot_single_datasets))
+
+supergpqa_0shot_single_datasets.append(supergpqa_dataset)