diff --git a/CHANGELOG.md b/CHANGELOG.md index 7760befc7a..7872d84240 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Add `VlmAd` metric by [Bepitic](https://github.com/Bepitic) and refactored by [ashwinvaidya17](https://github.com/ashwinvaidya17) in https://github.com/openvinotoolkit/anomalib/pull/2344 - Add `Datumaro` annotation format support by @ashwinvaidya17 in https://github.com/openvinotoolkit/anomalib/pull/2377 -- Add `AUPIMO` tutorials notebooks in https://github.com/openvinotoolkit/anomalib/pull/2330 and https://github.com/openvinotoolkit/anomalib/pull/2336 +- Add `AUPIMO` tutorials notebooks in https://github.com/openvinotoolkit/anomalib/pull/2330 , https://github.com/openvinotoolkit/anomalib/pull/2336 , https://github.com/openvinotoolkit/anomalib/pull/2347 , https://github.com/openvinotoolkit/anomalib/pull/2348 , https://github.com/openvinotoolkit/anomalib/pull/2352 and https://github.com/openvinotoolkit/anomalib/pull/2380 by [jpcbertoldo](https://github.com/jpcbertoldo) - Add `AUPIMO` metric by [jpcbertoldo](https://github.com/jpcbertoldo) in https://github.com/openvinotoolkit/anomalib/pull/1726 and refactored by [ashwinvaidya17](https://github.com/ashwinvaidya17) in https://github.com/openvinotoolkit/anomalib/pull/2329 ### Changed diff --git a/notebooks/700_metrics/701a_aupimo.ipynb b/notebooks/700_metrics/701a_aupimo.ipynb index d780c5a964..a85c92a92c 100644 --- a/notebooks/700_metrics/701a_aupimo.ipynb +++ b/notebooks/700_metrics/701a_aupimo.ipynb @@ -58,8 +58,7 @@ "metadata": {}, "outputs": [], "source": [ - "# TODO(jpcbertoldo): replace by `pip install anomalib` when AUPIMO is released # noqa: TD003\n", - "%pip install ../.." + "%pip install anomalib" ] }, { @@ -505,8 +504,15 @@ " year={2024},\n", " url={https://arxiv.org/abs/2401.01984}, \n", "}\n", - "```" + "```\n", + "\n", + "For more tutorials, see [github.com/openvinotoolkit/anomalib/tree/main/notebooks/700_metrics](https://github.com/openvinotoolkit/anomalib/tree/main/notebooks/700_metrics)." ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] } ], "metadata": { diff --git a/notebooks/700_metrics/701b_aupimo_advanced_i.ipynb b/notebooks/700_metrics/701b_aupimo_advanced_i.ipynb index ea322102f8..133982c549 100644 --- a/notebooks/700_metrics/701b_aupimo_advanced_i.ipynb +++ b/notebooks/700_metrics/701b_aupimo_advanced_i.ipynb @@ -6,13 +6,13 @@ "source": [ "# AUPIMO\n", "\n", - "Advance use cases of the metric AUPIMO (pronounced \"a-u-pee-mo\").\n", - "\n", - "> For basic usage, please check the notebook [701a_aupimo.ipynb](./701a_aupimo.ipynb).\n", + "Advanced use cases of the metric AUPIMO (pronounced \"a-u-pee-mo\").\n", "\n", "Includes:\n", - "- selection of test representative samples for qualitative analysis\n", - "- visualization of the AUPIMO metric with heatmaps" + "- selection of representative samples + util function (in the end)\n", + "- visualization of the AUPIMO with heatmaps\n", + "\n", + "For basic usage, please check the notebook [701a_aupimo.ipynb](./701a_aupimo.ipynb)." ] }, { @@ -64,8 +64,7 @@ "metadata": {}, "outputs": [], "source": [ - "# TODO(jpcbertoldo): replace by `pip install anomalib` when AUPIMO is released # noqa: TD003\n", - "%pip install ../.." + "%pip install anomalib" ] }, { @@ -788,7 +787,9 @@ " year={2024},\n", " url={https://arxiv.org/abs/2401.01984}, \n", "}\n", - "```" + "```\n", + "\n", + "For more tutorials, see [github.com/openvinotoolkit/anomalib/tree/main/notebooks/700_metrics](https://github.com/openvinotoolkit/anomalib/tree/main/notebooks/700_metrics)." ] }, { @@ -1386,7 +1387,9 @@ " year={2024},\n", " url={https://arxiv.org/abs/2401.01984}, \n", "}\n", - "```" + "```\n", + "\n", + "For more tutorials, see [github.com/openvinotoolkit/anomalib/tree/main/notebooks/700_metrics](https://github.com/openvinotoolkit/anomalib/tree/main/notebooks/700_metrics)." ] } ], diff --git a/notebooks/700_metrics/701c_aupimo_advanced_ii.ipynb b/notebooks/700_metrics/701c_aupimo_advanced_ii.ipynb index 6911b9c546..44470a9f6f 100644 --- a/notebooks/700_metrics/701c_aupimo_advanced_ii.ipynb +++ b/notebooks/700_metrics/701c_aupimo_advanced_ii.ipynb @@ -6,15 +6,13 @@ "source": [ "# AUPIMO\n", "\n", - "Advance use cases of the metric AUPIMO (pronounced \"a-u-pee-mo\").\n", - "\n", - "> For basic usage, please check the notebook [701a_aupimo.ipynb](./701a_aupimo.ipynb).\n", + "Advanced use cases of the metric AUPIMO (pronounced \"a-u-pee-mo\").\n", "\n", "Includes:\n", "- visualization of the PIMO curve\n", - "- theoretical AUPIMO of a random classifier (\"baseline\")\n", - "- understanding the x-axis (FPR) bounds\n", - "- customizing the x-axis (FPR) bounds" + "- understanding and customizing the x-axis (FPRn) bounds\n", + "\n", + "For basic usage, please check the notebook [701a_aupimo.ipynb](./701a_aupimo.ipynb)." ] }, { @@ -66,8 +64,7 @@ "metadata": {}, "outputs": [], "source": [ - "# TODO(jpcbertoldo): replace by `pip install anomalib` when AUPIMO is released # noqa: TD003\n", - "%pip install ../.." + "%pip install anomalib" ] }, { @@ -898,7 +895,9 @@ " year={2024},\n", " url={https://arxiv.org/abs/2401.01984}, \n", "}\n", - "```" + "```\n", + "\n", + "For more tutorials, see [github.com/openvinotoolkit/anomalib/tree/main/notebooks/700_metrics](https://github.com/openvinotoolkit/anomalib/tree/main/notebooks/700_metrics)." ] } ], diff --git a/notebooks/700_metrics/701d_aupimo_advanced_iii.ipynb b/notebooks/700_metrics/701d_aupimo_advanced_iii.ipynb index 7cbd29823b..46a0e749be 100644 --- a/notebooks/700_metrics/701d_aupimo_advanced_iii.ipynb +++ b/notebooks/700_metrics/701d_aupimo_advanced_iii.ipynb @@ -6,15 +6,15 @@ "source": [ "# AUPIMO Score of a Random Model\n", "\n", + "Theoretical properties of the metric AUPIMO (pronounced \"a-u-pee-mo\").\n", + "\n", "If model randomly assigns scores to the pixels -- i.e. no discrimination -- its AUROC score will be 50%. \n", "\n", "What would be its AUPIMO score?\n", "\n", - "> AUPIMO is pronounced \"a-u-pee-mo\".\n", - "\n", - "> For basic usage, please check the notebook [701a_aupimo.ipynb](./701a_aupimo.ipynb).\n", + "For basic usage, please check the notebook [701a_aupimo.ipynb](./701a_aupimo.ipynb).\n", "\n", - "> For PIMO curve plots, please check the notebook [701c_aupimo_advanced_ii.ipynb](./701c_aupimo_advanced_ii.ipynb)." + "For PIMO curve plots, please check the notebook [701c_aupimo_advanced_ii.ipynb](./701c_aupimo_advanced_ii.ipynb)." ] }, { @@ -66,8 +66,7 @@ "metadata": {}, "outputs": [], "source": [ - "# TODO(jpcbertoldo): replace by `pip install anomalib` when AUPIMO is released # noqa: TD003\n", - "%pip install ../.." + "%pip install anomalib" ] }, { @@ -334,7 +333,9 @@ " year={2024},\n", " url={https://arxiv.org/abs/2401.01984}, \n", "}\n", - "```" + "```\n", + "\n", + "For more tutorials, see [github.com/openvinotoolkit/anomalib/tree/main/notebooks/700_metrics](https://github.com/openvinotoolkit/anomalib/tree/main/notebooks/700_metrics)." ] } ], diff --git a/notebooks/700_metrics/README.md b/notebooks/700_metrics/README.md new file mode 100644 index 0000000000..0b2ca89f27 --- /dev/null +++ b/notebooks/700_metrics/README.md @@ -0,0 +1,35 @@ +# AUPIMO Tutorials + +| Notebook | GitHub | Colab | +| ----------------------------------------------- | --------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| AUPIMO basics | [701a_aupimo](/notebooks/700_metrics/701a_aupimo.ipynb) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/openvinotoolkit/anomalib/blob/main/notebooks/700_metrics/701a_aupimo.ipynb) | +| AUPIMO representative samples and visualization | [701b_aupimo_advanced_i](/notebooks/700_metrics/701b_aupimo_advanced_i.ipynb) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/openvinotoolkit/anomalib/blob/main/notebooks/700_metrics/701b_aupimo_advanced_i.ipynb) | +| PIMO curve and integration bounds | [701c_aupimo_advanced_ii](/notebooks/700_metrics/701c_aupimo_advanced_ii.ipynb) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/openvinotoolkit/anomalib/blob/main/notebooks/700_metrics/701c_aupimo_advanced_ii.ipynb) | +| (AU)PIMO of a random model | [701d_aupimo_advanced_iii](/notebooks/700_metrics/701d_aupimo_advanced_iii.ipynb) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/openvinotoolkit/anomalib/blob/main/notebooks/700_metrics/701d_aupimo_advanced_iii.ipynb) | +| AUPIMO load/save, statistical comparison | [701e_aupimo_advanced_iv](/notebooks/700_metrics/701e_aupimo_advanced_iv.ipynb) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/openvinotoolkit/anomalib/blob/main/notebooks/700_metrics/701e_aupimo_advanced_iv.ipynb) | + +## Installation Instructions + +If you have not installed all required dependencies, follow the [Installation Guide](https://openvinotoolkit.github.io/anomalib/getting_started/installation/index.html). + +## Notebook Contents + +### AUPIMO (701 series) + +The first notebook (`701a_aupimo`) introduces the basic usages of AUPIMO. It covers how to: + +- get the average AUPIMO in the `Engine` +- access all individual AUPIMO values using the torchmetrics API. + +The following notebooks show other advanced usages of AUPIMO, including how to: + +- (in `701b_aupimo_advanced_i`) + - select representative anomalous samples from the distribution of AUPIMO scores + - visualize AUPIMO with heatmaps +- (in `701c_aupimo_advanced_ii`) + - plot the PIMO curve and the integration bounds + - visualize the validation condition imposed by the integration bounds (i.e. what the FPRn bounds mean in practice) +- (in `701d_aupimo_advanced_iii`) + - a reference value for AUPIMO, calculated from a model that predicts random anomaly scores +- (in `701e_aupimo_advanced_iv`) + - TODO(jpcbertoldo): add description