From 2d5ea4e9567ce43a52dd485378e80ba27e5e4119 Mon Sep 17 00:00:00 2001 From: Nikos Livathinos Date: Mon, 24 Feb 2025 17:20:58 +0100 Subject: [PATCH] chore: Add CONVERTER_TYPE and MODALITIES columns to all produced datasets Signed-off-by: Nikos Livathinos --- docling_eval/benchmarks/cvat_annotation/create.py | 7 ++++++- docling_eval/benchmarks/doclaynet_v1/create.py | 10 +++++++++- docling_eval/benchmarks/dpbench/create.py | 13 ++++++++++++- docling_eval/benchmarks/funsd/create.py | 11 ++++++++++- docling_eval/benchmarks/omnidocbench/create.py | 11 ++++++++++- .../tableformer_huggingface_otsl/create.py | 10 +++++++++- 6 files changed, 56 insertions(+), 6 deletions(-) diff --git a/docling_eval/benchmarks/cvat_annotation/create.py b/docling_eval/benchmarks/cvat_annotation/create.py index 8336bd8..9c3c773 100644 --- a/docling_eval/benchmarks/cvat_annotation/create.py +++ b/docling_eval/benchmarks/cvat_annotation/create.py @@ -23,7 +23,11 @@ from PIL import Image # as PILImage from tqdm import tqdm # type: ignore -from docling_eval.benchmarks.constants import BenchMarkColumns, EvaluationModality +from docling_eval.benchmarks.constants import ( + BenchMarkColumns, + ConverterTypes, + EvaluationModality, +) from docling_eval.benchmarks.cvat_annotation.utils import ( AnnotatedImage, AnnotationOverview, @@ -979,6 +983,7 @@ def create_layout_dataset_from_annotations( ) record = { + BenchMarkColumns.CONVERTER_TYPE: ConverterTypes.DOCLING, BenchMarkColumns.DOCLING_VERSION: docling_version(), BenchMarkColumns.STATUS: str(conv_results.status), BenchMarkColumns.DOC_ID: str(basename), diff --git a/docling_eval/benchmarks/doclaynet_v1/create.py b/docling_eval/benchmarks/doclaynet_v1/create.py index 9495892..4152466 100644 --- a/docling_eval/benchmarks/doclaynet_v1/create.py +++ b/docling_eval/benchmarks/doclaynet_v1/create.py @@ -21,7 +21,11 @@ from docling_core.types.io import DocumentStream from tqdm import tqdm # type: ignore -from docling_eval.benchmarks.constants import BenchMarkColumns, ConverterTypes +from docling_eval.benchmarks.constants import ( + BenchMarkColumns, + ConverterTypes, + EvaluationModality, +) from docling_eval.benchmarks.utils import add_pages_to_true_doc, write_datasets_info from docling_eval.converters.conversion import ( create_docling_converter, @@ -265,6 +269,10 @@ def create_dlnv1_e2e_dataset( BenchMarkColumns.PREDICTION_PICTURES: pred_pictures, BenchMarkColumns.ORIGINAL: pdf_stream.getvalue(), BenchMarkColumns.MIMETYPE: "image/png", + BenchMarkColumns.MODALITIES: [ + EvaluationModality.LAYOUT, + EvaluationModality.READING_ORDER, + ], } pdf_stream.close() records.append(record) diff --git a/docling_eval/benchmarks/dpbench/create.py b/docling_eval/benchmarks/dpbench/create.py index 3145eaf..9ffa0b5 100644 --- a/docling_eval/benchmarks/dpbench/create.py +++ b/docling_eval/benchmarks/dpbench/create.py @@ -28,7 +28,11 @@ from docling_core.types.doc.labels import DocItemLabel from PIL import Image # as PILImage -from docling_eval.benchmarks.constants import BenchMarkColumns, ConverterTypes +from docling_eval.benchmarks.constants import ( + BenchMarkColumns, + ConverterTypes, + EvaluationModality, +) from docling_eval.benchmarks.utils import ( add_pages_to_true_doc, convert_html_table_into_docling_tabledata, @@ -341,6 +345,10 @@ def create_dpbench_e2e_dataset( BenchMarkColumns.PREDICTION_PICTURES: pred_pictures, BenchMarkColumns.ORIGINAL: get_binary(pdf_path), BenchMarkColumns.MIMETYPE: "application/pdf", + BenchMarkColumns.MODALITIES: [ + EvaluationModality.LAYOUT, + EvaluationModality.READING_ORDER, + ], } records.append(record) @@ -448,6 +456,9 @@ def create_dpbench_tableformer_dataset( BenchMarkColumns.GROUNDTRUTH_PICTURES: true_pictures, BenchMarkColumns.PREDICTION_PAGE_IMAGES: pred_page_images, BenchMarkColumns.PREDICTION_PICTURES: pred_pictures, + BenchMarkColumns.MODALITIES: [ + EvaluationModality.TABLE_STRUCTURE, + ], } records.append(record) diff --git a/docling_eval/benchmarks/funsd/create.py b/docling_eval/benchmarks/funsd/create.py index 47e6ec9..d5d504a 100644 --- a/docling_eval/benchmarks/funsd/create.py +++ b/docling_eval/benchmarks/funsd/create.py @@ -26,7 +26,11 @@ from PIL import Image from tqdm import tqdm # type: ignore -from docling_eval.benchmarks.constants import BenchMarkColumns +from docling_eval.benchmarks.constants import ( + BenchMarkColumns, + ConverterTypes, + EvaluationModality, +) from docling_eval.benchmarks.utils import write_datasets_info from docling_eval.converters.conversion import create_image_converter from docling_eval.converters.utils import ( @@ -511,6 +515,7 @@ def create_funsd_dataset( ) record = { + BenchMarkColumns.CONVERTER_TYPE: ConverterTypes.DOCLING, BenchMarkColumns.DOCLING_VERSION: docling_version(), BenchMarkColumns.DOC_ID: img_path.stem, BenchMarkColumns.GROUNDTRUTH: json.dumps(true_doc.export_to_dict()), @@ -518,6 +523,10 @@ def create_funsd_dataset( BenchMarkColumns.GROUNDTRUTH_PICTURES: true_pictures, BenchMarkColumns.ORIGINAL: img_bytes, BenchMarkColumns.MIMETYPE: "image/png", + BenchMarkColumns.MODALITIES: [ + EvaluationModality.LAYOUT, + EvaluationModality.READING_ORDER, + ], } records.append(record) count += 1 diff --git a/docling_eval/benchmarks/omnidocbench/create.py b/docling_eval/benchmarks/omnidocbench/create.py index 0badba1..0288bd1 100644 --- a/docling_eval/benchmarks/omnidocbench/create.py +++ b/docling_eval/benchmarks/omnidocbench/create.py @@ -12,7 +12,11 @@ from PIL import Image # as PILImage from tqdm import tqdm # type: ignore -from docling_eval.benchmarks.constants import BenchMarkColumns, ConverterTypes +from docling_eval.benchmarks.constants import ( + BenchMarkColumns, + ConverterTypes, + EvaluationModality, +) from docling_eval.benchmarks.utils import ( add_pages_to_true_doc, convert_html_table_into_docling_tabledata, @@ -368,6 +372,10 @@ def create_omnidocbench_e2e_dataset( BenchMarkColumns.PREDICTION_PICTURES: pred_pictures, BenchMarkColumns.GROUNDTRUTH_PAGE_IMAGES: true_page_images, BenchMarkColumns.GROUNDTRUTH_PICTURES: true_pictures, + BenchMarkColumns.MODALITIES: [ + EvaluationModality.LAYOUT, + EvaluationModality.READING_ORDER, + ], } records.append(record) @@ -485,6 +493,7 @@ def create_omnidocbench_tableformer_dataset( BenchMarkColumns.PREDICTION_PICTURES: pred_pictures, BenchMarkColumns.GROUNDTRUTH_PAGE_IMAGES: true_page_images, BenchMarkColumns.GROUNDTRUTH_PICTURES: true_pictures, + BenchMarkColumns.MODALITIES: [EvaluationModality.TABLE_STRUCTURE], } records.append(record) diff --git a/docling_eval/benchmarks/tableformer_huggingface_otsl/create.py b/docling_eval/benchmarks/tableformer_huggingface_otsl/create.py index 2fbff79..a26269f 100644 --- a/docling_eval/benchmarks/tableformer_huggingface_otsl/create.py +++ b/docling_eval/benchmarks/tableformer_huggingface_otsl/create.py @@ -17,7 +17,11 @@ from docling_core.types.doc.labels import DocItemLabel from tqdm import tqdm # type: ignore -from docling_eval.benchmarks.constants import BenchMarkColumns +from docling_eval.benchmarks.constants import ( + BenchMarkColumns, + ConverterTypes, + EvaluationModality, +) from docling_eval.benchmarks.utils import convert_html_table_into_docling_tabledata from docling_eval.converters.models.tableformer.tf_model_prediction import ( PageTokens, @@ -221,6 +225,7 @@ def create_huggingface_otsl_tableformer_dataset( ) record = { + BenchMarkColumns.CONVERTER_TYPE: ConverterTypes.DOCLING, BenchMarkColumns.DOCLING_VERSION: docling_version(), BenchMarkColumns.STATUS: str(ConversionStatus.SUCCESS.value), BenchMarkColumns.DOC_ID: str(os.path.basename(filename)), @@ -232,10 +237,12 @@ def create_huggingface_otsl_tableformer_dataset( BenchMarkColumns.PREDICTION_PICTURES: pred_pictures, BenchMarkColumns.GROUNDTRUTH_PAGE_IMAGES: true_page_images, BenchMarkColumns.GROUNDTRUTH_PICTURES: true_pictures, + BenchMarkColumns.MODALITIES: [EvaluationModality.TABLE_STRUCTURE], } records.append(record) else: record = { + BenchMarkColumns.CONVERTER_TYPE: ConverterTypes.DOCLING, BenchMarkColumns.DOCLING_VERSION: docling_version(), BenchMarkColumns.STATUS: str(ConversionStatus.FAILURE.value), BenchMarkColumns.DOC_ID: str(os.path.basename(filename)), @@ -247,6 +254,7 @@ def create_huggingface_otsl_tableformer_dataset( BenchMarkColumns.PREDICTION_PICTURES: pred_pictures, BenchMarkColumns.GROUNDTRUTH_PAGE_IMAGES: true_page_images, BenchMarkColumns.GROUNDTRUTH_PICTURES: true_pictures, + BenchMarkColumns.MODALITIES: [EvaluationModality.TABLE_STRUCTURE], } records.append(record)