updated the README

Signed-off-by: Peter Staar <[email protected]>
DS4SD · Dec 20, 2024 · 0a6829d · 0a6829d
1 parent 236c9da
commit 0a6829d
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 8 deletions.
diff --git a/README.md b/README.md
@@ -26,7 +26,7 @@ docling-eval % poetry run evaluate --help
  Usage: evaluate [OPTIONS]
 
 ╭─ Options ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
-│ *  --task        -t      [create|evaluate]                                                                          Evaluation task [default: None] [required]                                                                              │
+│ *  --task        -t      [create|evaluate|visualize]                                                                Evaluation task [default: None] [required]                                                                              │
 │ *  --modality    -m      [end-to-end|layout|tableformer|codeformer]                                                 Evaluation modality [default: None] [required]                                                                          │
 │ *  --benchmark   -b      [DPBench|OmniDcoBench|WordScape|PubLayNet|DocLayNet|Pub1M|PubTabNet|FinTabNet|WikiTabNet]  Benchmark name [default: None] [required]                                                                               │
 │ *  --input-dir   -i      PATH                                                                                       Input directory [default: None] [required]                                                                              │
@@ -35,18 +35,31 @@ docling-eval % poetry run evaluate --help
 ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
 ```
 
-If you have DP-Bench locally, you can create the dataset,
+## End to End examples
+
+### Table evaluations for DP-Bench
+
+👉 Create the dataset,
 
 ```sh
-poetry run evaluate -t create -m end-to-end -b DPBench -i <location-of-dpbench> -o ./benchmarks/dpbench-tableformer
+poetry run evaluate -t create -m tableformer -b DPBench -i <location-of-dpbench> -o ./benchmarks/dpbench-tableformer
 ```
 
-or,
+👉 Evaluate the dataset,
 
 ```sh
-poetry run evaluate -t create -m tableformer -b DPBench -i <location-of-dpbench> -o ./benchmarks/dpbench-tableformer
+poetry run evaluate -t evaluate -m tableformer -b DPBench -i <location-of-dpbench> -o ./benchmarks/dpbench-tableformer
 ```
 
+👉 Visualise the dataset,
+
+```sh
+poetry run evaluate -t visualize -m tableformer -b DPBench -i <location-of-dpbench> -o ./benchmarks/dpbench-tableformer
+```
+
+The final result can be visualised as,
+
+[DPBench_TEDS](./docs/evaluations/evaluation_DPBench_tableformer.pdf)
 
 
 ## Contributing

diff --git a/docling_eval/cli/main.py b/docling_eval/cli/main.py
@@ -107,18 +107,22 @@ def visualise(modality:EvaluationModality, benchmark:BenchMarkNames, idir:Path,
 
             # Calculate bin widths
             bin_widths = [evaluation.TEDS.bins[i + 1] - evaluation.TEDS.bins[i] for i in range(len(evaluation.TEDS.bins) - 1)]
-
+            bin_middle = [(evaluation.TEDS.bins[i + 1] + evaluation.TEDS.bins[i])/2.0 for i in range(len(evaluation.TEDS.bins) - 1)]
+
             for i in range(len(evaluation.TEDS.bins)-1):
                 logging.info(f"{i:02} [{evaluation.TEDS.bins[i]:.3f}, {evaluation.TEDS.bins[i+1]:.3f}]: {evaluation.TEDS.hist[i]}")
 
             # Plot histogram
-            plt.bar(evaluation.TEDS.bins[0:-1], evaluation.TEDS.hist, width=bin_widths, edgecolor="black")
+            plt.bar(bin_middle, evaluation.TEDS.hist, width=bin_widths, edgecolor="black")
             #width=(evaluation.TEDS.bins[1] - evaluation.TEDS.bins[0]),
 
             plt.xlabel("TEDS")
             plt.ylabel("Frequency")
             plt.title(f"benchmark: {benchmark.value}, modality: {modality.value}")
-            plt.show()
+
+            figname = odir / f"evaluation_{benchmark.value}_{modality.value}.pdf"
+            logging.info(f"saving figure to {figname}")
+            plt.savefig(figname)
 
         case EvaluationModality.CODEFORMER:
             pass

diff --git a/docs/evaluations/evaluation_DPBench_tableformer.pdf b/docs/evaluations/evaluation_DPBench_tableformer.pdf