From b8dc4cd9707621cbe73f51407180c386b7d830f9 Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Fri, 7 Mar 2025 08:38:13 +1000 Subject: [PATCH 1/8] feat(api): add util to extract metadata from image --- .../app/api/extract_metadata_from_image.py | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 invokeai/app/api/extract_metadata_from_image.py diff --git a/invokeai/app/api/extract_metadata_from_image.py b/invokeai/app/api/extract_metadata_from_image.py new file mode 100644 index 00000000000..5153b524f22 --- /dev/null +++ b/invokeai/app/api/extract_metadata_from_image.py @@ -0,0 +1,108 @@ +import json +import logging +from dataclasses import dataclass + +from PIL import Image + +from invokeai.app.services.shared.graph import Graph +from invokeai.app.services.workflow_records.workflow_records_common import WorkflowWithoutIDValidator + + +@dataclass +class ExtractedMetadata: + invokeai_metadata: str | None + invokeai_workflow: str | None + invokeai_graph: str | None + + +def extract_metadata_from_image( + pil_image: Image.Image, + invokeai_metadata_override: str | None, + invokeai_workflow_override: str | None, + invokeai_graph_override: str | None, + logger: logging.Logger, +) -> ExtractedMetadata: + """ + Extracts the "invokeai_metadata", "invokeai_workflow", and "invokeai_graph" data embedded in the PIL Image. + + These items are stored as stringified JSON in the image file's metadata, so we need to do some parsing to validate + them. Once parsed, the values are returned as they came (as strings), or None if they are not present or invalid. + + In some situations, we may prefer to override the values extracted from the image file with some other values. + + For example, when uploading an image via API, the client can optionally provide the metadata directly in the request, + as opposed to embedding it in the image file. In this case, the client-provided metadata will be used instead of the + metadata embedded in the image file. + + Args: + pil_image: The PIL Image object. + invokeai_metadata_override: The metadata override provided by the client. + invokeai_workflow_override: The workflow override provided by the client. + invokeai_graph_override: The graph override provided by the client. + logger: The logger to use for debug logging. + + Returns: + ExtractedMetadata: The extracted metadata, workflow, and graph. + """ + + # The fallback value for metadata is None. + stringified_metadata: str | None = None + + # Use the metadata override if provided, else attempt to extract it from the image file. + metadata_raw = invokeai_metadata_override or pil_image.info.get("invokeai_metadata", None) + + # If the metadata is present in the image file, we will attempt to parse it as JSON. When we create images, + # we always store metadata as a stringified JSON dict. So, we expect it to be a string here. + if isinstance(metadata_raw, str): + try: + # Must be a JSON string + metadata_parsed = json.loads(metadata_raw) + # Must be a dict + if isinstance(metadata_parsed, dict): + # Looks good, overwrite the fallback value + stringified_metadata = metadata_raw + except Exception as e: + logger.debug(f"Failed to parse metadata for uploaded image, {e}") + pass + + # We expect the workflow, if embedded in the image, to be a JSON-stringified WorkflowWithoutID. We will store it + # as a string. + workflow_raw: str | None = invokeai_workflow_override or pil_image.info.get("invokeai_workflow", None) + + # The fallback value for workflow is None. + stringified_workflow: str | None = None + + # If the workflow is present in the image file, we will attempt to parse it as JSON. When we create images, we + # always store workflows as a stringified JSON WorkflowWithoutID. So, we expect it to be a string here. + if isinstance(workflow_raw, str): + try: + # Validate the workflow JSON before storing it + WorkflowWithoutIDValidator.validate_json(workflow_raw) + # Looks good, overwrite the fallback value + stringified_workflow = workflow_raw + except Exception: + logger.debug("Failed to parse workflow for uploaded image") + pass + + # We expect the workflow, if embedded in the image, to be a JSON-stringified Graph. We will store it as a + # string. + graph_raw: str | None = invokeai_graph_override or pil_image.info.get("invokeai_graph", None) + + # The fallback value for graph is None. + stringified_graph: str | None = None + + # If the graph is present in the image file, we will attempt to parse it as JSON. When we create images, we + # always store graphs as a stringified JSON Graph. So, we expect it to be a string here. + if isinstance(graph_raw, str): + try: + # Validate the graph JSON before storing it + Graph.model_validate_json(graph_raw) + # Looks good, overwrite the fallback value + stringified_graph = graph_raw + except Exception as e: + logger.debug(f"Failed to parse graph for uploaded image, {e}") + pass + + return ExtractedMetadata( + invokeai_metadata=stringified_metadata, invokeai_workflow=stringified_workflow, invokeai_graph=stringified_graph + ) From 366ae620f85aff8cc41ae15c4ce4b1ff9976294a Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Fri, 7 Mar 2025 08:39:35 +1000 Subject: [PATCH 2/8] tests: add tests for extract_metadata_from_image --- tests/app/test_extract_metadata_from_image.py | 204 ++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 tests/app/test_extract_metadata_from_image.py diff --git a/tests/app/test_extract_metadata_from_image.py b/tests/app/test_extract_metadata_from_image.py new file mode 100644 index 00000000000..3c028780e9e --- /dev/null +++ b/tests/app/test_extract_metadata_from_image.py @@ -0,0 +1,204 @@ +import json +import logging +from unittest.mock import MagicMock, patch + +import pytest +from PIL import Image + +from invokeai.app.api.extract_metadata_from_image import ExtractedMetadata, extract_metadata_from_image + + +@pytest.fixture +def mock_logger(): + return MagicMock(spec=logging.Logger) + + +@pytest.fixture +def valid_metadata(): + return json.dumps({"param1": "value1", "param2": 123}) + + +@pytest.fixture +def valid_workflow(): + return json.dumps({"name": "test_workflow", "version": "1.0"}) + + +@pytest.fixture +def valid_graph(): + return json.dumps({"nodes": [], "edges": []}) + + +def test_extract_valid_metadata_from_image(mock_logger, valid_metadata, valid_workflow, valid_graph): + # Create a mock image with valid metadata + mock_image = MagicMock(spec=Image.Image) + mock_image.info = { + "invokeai_metadata": valid_metadata, + "invokeai_workflow": valid_workflow, + "invokeai_graph": valid_graph, + } + + # Mock the validation functions + with patch( + "invokeai.app.services.workflow_records.workflow_records_common.WorkflowWithoutIDValidator.validate_json" + ) as mock_workflow_validate: + with patch("invokeai.app.services.shared.graph.Graph.model_validate_json") as mock_graph_validate: + result = extract_metadata_from_image(mock_image, None, None, None, mock_logger) + + # Assert correct calls to validators + mock_workflow_validate.assert_called_once_with(valid_workflow) + mock_graph_validate.assert_called_once_with(valid_graph) + + # Assert correct extraction + assert result == ExtractedMetadata( + invokeai_metadata=valid_metadata, invokeai_workflow=valid_workflow, invokeai_graph=valid_graph + ) + + +def test_extract_invalid_metadata(mock_logger, valid_workflow, valid_graph): + # Invalid metadata (not JSON) + invalid_metadata = "not a valid json" + + mock_image = MagicMock(spec=Image.Image) + mock_image.info = { + "invokeai_metadata": invalid_metadata, + "invokeai_workflow": valid_workflow, + "invokeai_graph": valid_graph, + } + + with patch( + "invokeai.app.services.workflow_records.workflow_records_common.WorkflowWithoutIDValidator.validate_json" + ): + with patch("invokeai.app.services.shared.graph.Graph.model_validate_json"): + result = extract_metadata_from_image(mock_image, None, None, None, mock_logger) + + assert mock_logger.debug.to_have_been_called_with("Failed to parse metadata for uploaded image") + + # Invalid metadata should be None, others valid + assert result.invokeai_metadata is None + assert result.invokeai_workflow == valid_workflow + assert result.invokeai_graph == valid_graph + + +def test_metadata_wrong_type(mock_logger, valid_workflow, valid_graph): + # Valid JSON but not a dict + metadata_array = json.dumps(["item1", "item2"]) + + mock_image = MagicMock(spec=Image.Image) + mock_image.info = { + "invokeai_metadata": metadata_array, + "invokeai_workflow": valid_workflow, + "invokeai_graph": valid_graph, + } + + with patch( + "invokeai.app.services.workflow_records.workflow_records_common.WorkflowWithoutIDValidator.validate_json" + ): + with patch("invokeai.app.services.shared.graph.Graph.model_validate_json"): + result = extract_metadata_from_image(mock_image, None, None, None, mock_logger) + + # Metadata should be None as it's not a dict + assert result.invokeai_metadata is None + assert result.invokeai_workflow == valid_workflow + assert result.invokeai_graph == valid_graph + + +def test_with_non_string_metadata(mock_logger, valid_workflow, valid_graph): + # Some implementations might include metadata as non-string values + mock_image = MagicMock(spec=Image.Image) + mock_image.info = { + "invokeai_metadata": 12345, # Not a string + "invokeai_workflow": valid_workflow, + "invokeai_graph": valid_graph, + } + + with patch( + "invokeai.app.services.workflow_records.workflow_records_common.WorkflowWithoutIDValidator.validate_json" + ): + with patch("invokeai.app.services.shared.graph.Graph.model_validate_json"): + result = extract_metadata_from_image(mock_image, None, None, None, mock_logger) + + assert mock_logger.debug.to_have_been_called_with("Failed to parse metadata for uploaded image") + + assert result.invokeai_metadata is None + assert result.invokeai_workflow == valid_workflow + assert result.invokeai_graph == valid_graph + + +def test_invalid_workflow(mock_logger, valid_metadata, valid_graph): + invalid_workflow = "not a valid workflow json" + + mock_image = MagicMock(spec=Image.Image) + mock_image.info = { + "invokeai_metadata": valid_metadata, + "invokeai_workflow": invalid_workflow, + "invokeai_graph": valid_graph, + } + + with patch( + "invokeai.app.services.workflow_records.workflow_records_common.WorkflowWithoutIDValidator.validate_json" + ) as mock_validate: + mock_validate.side_effect = ValueError("Invalid workflow") + with patch("invokeai.app.services.shared.graph.Graph.model_validate_json"): + result = extract_metadata_from_image(mock_image, None, None, None, mock_logger) + + assert result.invokeai_metadata == valid_metadata + assert result.invokeai_workflow is None + assert result.invokeai_graph == valid_graph + + +def test_invalid_graph(mock_logger, valid_metadata, valid_workflow): + invalid_graph = "not a valid graph json" + + mock_image = MagicMock(spec=Image.Image) + mock_image.info = { + "invokeai_metadata": valid_metadata, + "invokeai_workflow": valid_workflow, + "invokeai_graph": invalid_graph, + } + + with patch( + "invokeai.app.services.workflow_records.workflow_records_common.WorkflowWithoutIDValidator.validate_json" + ): + with patch("invokeai.app.services.shared.graph.Graph.model_validate_json") as mock_validate: + mock_validate.side_effect = ValueError("Invalid graph") + result = extract_metadata_from_image(mock_image, None, None, None, mock_logger) + + assert result.invokeai_metadata == valid_metadata + assert result.invokeai_workflow == valid_workflow + assert result.invokeai_graph is None + + +def test_with_overrides(mock_logger, valid_metadata, valid_workflow, valid_graph): + # Different values in the image + mock_image = MagicMock(spec=Image.Image) + + # When overrides are provided, they should be used instead of the values in the image, we shouldn'teven try + # to parse the values in the image + mock_image.info = { + "invokeai_metadata": 12345, + "invokeai_workflow": 12345, + "invokeai_graph": 12345, + } + + with patch( + "invokeai.app.services.workflow_records.workflow_records_common.WorkflowWithoutIDValidator.validate_json" + ): + with patch("invokeai.app.services.shared.graph.Graph.model_validate_json"): + result = extract_metadata_from_image(mock_image, valid_metadata, valid_workflow, valid_graph, mock_logger) + + # Override values should be used + assert result.invokeai_metadata == valid_metadata + assert result.invokeai_workflow == valid_workflow + assert result.invokeai_graph == valid_graph + + +def test_with_no_metadata(mock_logger): + # Image with no metadata + mock_image = MagicMock(spec=Image.Image) + mock_image.info = {} + + result = extract_metadata_from_image(mock_image, None, None, None, mock_logger) + + assert result.invokeai_metadata is None + assert result.invokeai_workflow is None + assert result.invokeai_graph is None From 0c02d626a9759cb6cda328f7d56a2c229ca08fde Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Fri, 7 Mar 2025 08:39:44 +1000 Subject: [PATCH 3/8] feat(api): use extract_metadata_from_image in upload router --- invokeai/app/api/routers/images.py | 50 ++++++++++-------------------- 1 file changed, 16 insertions(+), 34 deletions(-) diff --git a/invokeai/app/api/routers/images.py b/invokeai/app/api/routers/images.py index 14652ea7848..c86b554f9a0 100644 --- a/invokeai/app/api/routers/images.py +++ b/invokeai/app/api/routers/images.py @@ -6,9 +6,10 @@ from fastapi.responses import FileResponse from fastapi.routing import APIRouter from PIL import Image -from pydantic import BaseModel, Field, JsonValue +from pydantic import BaseModel, Field from invokeai.app.api.dependencies import ApiDependencies +from invokeai.app.api.extract_metadata_from_image import extract_metadata_from_image from invokeai.app.invocations.fields import MetadataField from invokeai.app.services.image_records.image_records_common import ( ImageCategory, @@ -45,18 +46,16 @@ async def upload_image( board_id: Optional[str] = Query(default=None, description="The board to add this image to, if any"), session_id: Optional[str] = Query(default=None, description="The session ID associated with this upload, if any"), crop_visible: Optional[bool] = Query(default=False, description="Whether to crop the image"), - metadata: Optional[JsonValue] = Body( - default=None, description="The metadata to associate with the image", embed=True + metadata: Optional[str] = Body( + default=None, + description="The metadata to associate with the image, must be a stringified JSON dict", + embed=True, ), ) -> ImageDTO: """Uploads an image""" if not file.content_type or not file.content_type.startswith("image"): raise HTTPException(status_code=415, detail="Not an image") - _metadata = None - _workflow = None - _graph = None - contents = await file.read() try: pil_image = Image.open(io.BytesIO(contents)) @@ -67,30 +66,13 @@ async def upload_image( ApiDependencies.invoker.services.logger.error(traceback.format_exc()) raise HTTPException(status_code=415, detail="Failed to read image") - # TODO: retain non-invokeai metadata on upload? - # attempt to parse metadata from image - metadata_raw = metadata if isinstance(metadata, str) else pil_image.info.get("invokeai_metadata", None) - if isinstance(metadata_raw, str): - _metadata = metadata_raw - else: - ApiDependencies.invoker.services.logger.debug("Failed to parse metadata for uploaded image") - pass - - # attempt to parse workflow from image - workflow_raw = pil_image.info.get("invokeai_workflow", None) - if isinstance(workflow_raw, str): - _workflow = workflow_raw - else: - ApiDependencies.invoker.services.logger.debug("Failed to parse workflow for uploaded image") - pass - - # attempt to extract graph from image - graph_raw = pil_image.info.get("invokeai_graph", None) - if isinstance(graph_raw, str): - _graph = graph_raw - else: - ApiDependencies.invoker.services.logger.debug("Failed to parse graph for uploaded image") - pass + extracted_metadata = extract_metadata_from_image( + pil_image=pil_image, + invokeai_metadata_override=metadata, + invokeai_workflow_override=None, + invokeai_graph_override=None, + logger=ApiDependencies.invoker.services.logger, + ) try: image_dto = ApiDependencies.invoker.services.images.create( @@ -99,9 +81,9 @@ async def upload_image( image_category=image_category, session_id=session_id, board_id=board_id, - metadata=_metadata, - workflow=_workflow, - graph=_graph, + metadata=extracted_metadata.invokeai_metadata, + workflow=extracted_metadata.invokeai_workflow, + graph=extracted_metadata.invokeai_graph, is_intermediate=is_intermediate, ) From 288f918e1d459b2439518f1d9f831dab7737bf52 Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Fri, 7 Mar 2025 08:39:51 +1000 Subject: [PATCH 4/8] chore(ui): typegen --- invokeai/frontend/web/src/services/api/schema.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/invokeai/frontend/web/src/services/api/schema.ts b/invokeai/frontend/web/src/services/api/schema.ts index 94539863336..f82a098e95d 100644 --- a/invokeai/frontend/web/src/services/api/schema.ts +++ b/invokeai/frontend/web/src/services/api/schema.ts @@ -2455,8 +2455,11 @@ export type components = { * Format: binary */ file: Blob; - /** @description The metadata to associate with the image */ - metadata?: components["schemas"]["JsonValue"] | null; + /** + * Metadata + * @description The metadata to associate with the image + */ + metadata?: string | null; }; /** * Boolean Collection Primitive From 1c8860091f92eeb1fbc8d8bde395cf2a0f906046 Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Fri, 7 Mar 2025 08:56:16 +1000 Subject: [PATCH 5/8] chore(ui): typegen --- invokeai/frontend/web/src/services/api/schema.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/invokeai/frontend/web/src/services/api/schema.ts b/invokeai/frontend/web/src/services/api/schema.ts index f82a098e95d..59efc634715 100644 --- a/invokeai/frontend/web/src/services/api/schema.ts +++ b/invokeai/frontend/web/src/services/api/schema.ts @@ -2457,7 +2457,7 @@ export type components = { file: Blob; /** * Metadata - * @description The metadata to associate with the image + * @description The metadata to associate with the image, must be a stringified JSON dict */ metadata?: string | null; }; From 656ed6bee957df23543f93d1c47b1485a03c503e Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Fri, 7 Mar 2025 10:10:42 +1000 Subject: [PATCH 6/8] fix(api): loosen graph parsing in extract_metadata_from_image There's a pydantic thing that causes the graphs to fail validation erroneously. Details in the comments - not a high priority to fix but we should figure it out someday. --- .../app/api/extract_metadata_from_image.py | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/invokeai/app/api/extract_metadata_from_image.py b/invokeai/app/api/extract_metadata_from_image.py index 5153b524f22..054b3cc38cc 100644 --- a/invokeai/app/api/extract_metadata_from_image.py +++ b/invokeai/app/api/extract_metadata_from_image.py @@ -4,7 +4,6 @@ from PIL import Image -from invokeai.app.services.shared.graph import Graph from invokeai.app.services.workflow_records.workflow_records_common import WorkflowWithoutIDValidator @@ -95,8 +94,25 @@ def extract_metadata_from_image( # always store graphs as a stringified JSON Graph. So, we expect it to be a string here. if isinstance(graph_raw, str): try: - # Validate the graph JSON before storing it - Graph.model_validate_json(graph_raw) + # TODO(psyche): Due to pydantic's handling of None values, it is possible for the graph to fail validation, + # even if it is a direct dump of a valid graph. Node fields in the graph are allowed to have be unset if + # they have incoming connections, but something about the ser/de process cannot adequately handle this. + # + # In lieu of fixing the graph validation, we will just do a simple check here to see if the graph is dict + # with the correct keys. This is not a perfect solution, but it should be good enough for now. + + # FIX ME: Validate the graph JSON before storing it + # Graph.model_validate_json(graph_raw) + + # Crappy workaround to validate JSON + graph_parsed = json.loads(graph_raw) + if not isinstance(graph_parsed, dict): + raise ValueError("Not a dict") + if not isinstance(graph_parsed.get("nodes", None), dict): + raise ValueError("'nodes' is not a dict") + if not isinstance(graph_parsed.get("edges", None), list): + raise ValueError("'edges' is not a list") + # Looks good, overwrite the fallback value stringified_graph = graph_raw except Exception as e: From 0591def2efd4377d4e2292e33b985e0a5dc8663f Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Fri, 7 Mar 2025 10:22:16 +1000 Subject: [PATCH 7/8] tests: fix test_extract_valid_metadata_from_image to accomodate prev commit --- tests/app/test_extract_metadata_from_image.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/app/test_extract_metadata_from_image.py b/tests/app/test_extract_metadata_from_image.py index 3c028780e9e..75ccac615f4 100644 --- a/tests/app/test_extract_metadata_from_image.py +++ b/tests/app/test_extract_metadata_from_image.py @@ -25,7 +25,7 @@ def valid_workflow(): @pytest.fixture def valid_graph(): - return json.dumps({"nodes": [], "edges": []}) + return json.dumps({"nodes": {}, "edges": []}) def test_extract_valid_metadata_from_image(mock_logger, valid_metadata, valid_workflow, valid_graph): @@ -46,7 +46,9 @@ def test_extract_valid_metadata_from_image(mock_logger, valid_metadata, valid_wo # Assert correct calls to validators mock_workflow_validate.assert_called_once_with(valid_workflow) - mock_graph_validate.assert_called_once_with(valid_graph) + # TODO(psyche): The extract_metadata_from_image does not validate the graph correctly. See note in `extract_metadata_from_image.py`. + # Skipping this. + # mock_graph_validate.assert_called_once_with(valid_graph) # Assert correct extraction assert result == ExtractedMetadata( From b94e8b817b686588d037e141b9b11377a3c7f7a3 Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Fri, 7 Mar 2025 10:24:15 +1000 Subject: [PATCH 8/8] chore: ruff --- tests/app/test_extract_metadata_from_image.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/app/test_extract_metadata_from_image.py b/tests/app/test_extract_metadata_from_image.py index 75ccac615f4..949d6df1656 100644 --- a/tests/app/test_extract_metadata_from_image.py +++ b/tests/app/test_extract_metadata_from_image.py @@ -41,14 +41,14 @@ def test_extract_valid_metadata_from_image(mock_logger, valid_metadata, valid_wo with patch( "invokeai.app.services.workflow_records.workflow_records_common.WorkflowWithoutIDValidator.validate_json" ) as mock_workflow_validate: - with patch("invokeai.app.services.shared.graph.Graph.model_validate_json") as mock_graph_validate: + with patch("invokeai.app.services.shared.graph.Graph.model_validate_json") as _mock_graph_validate: result = extract_metadata_from_image(mock_image, None, None, None, mock_logger) # Assert correct calls to validators mock_workflow_validate.assert_called_once_with(valid_workflow) # TODO(psyche): The extract_metadata_from_image does not validate the graph correctly. See note in `extract_metadata_from_image.py`. # Skipping this. - # mock_graph_validate.assert_called_once_with(valid_graph) + # _mock_graph_validate.assert_called_once_with(valid_graph) # Assert correct extraction assert result == ExtractedMetadata(