From b601006c8468c7e1a2759bb7558a07ccca0d827d Mon Sep 17 00:00:00 2001 From: Anirudh Date: Fri, 22 Nov 2024 16:48:10 -0500 Subject: [PATCH 1/3] fix: import enforce_response_format in api.py --- examples/interior_design_assistant/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/interior_design_assistant/api.py b/examples/interior_design_assistant/api.py index 5858df220..a2f3a4728 100644 --- a/examples/interior_design_assistant/api.py +++ b/examples/interior_design_assistant/api.py @@ -10,7 +10,7 @@ import uuid from pathlib import Path from typing import List - +from examples.interior_design_assistant.utils import enforce_response_format import fire from examples.interior_design_assistant.utils import ( @@ -168,7 +168,7 @@ async def suggest_alternatives( result = turn.output_message.content print(result) - return [r["description"].strip() for r in json.loads(result.strip())] + return enforce_response_format(result, n) async def retrieve_images(self, description: str) -> List[ImageMedia]: """ From f7fdf0ee69b56e3ae9bc5cdffe6ebeb9397c0cf5 Mon Sep 17 00:00:00 2001 From: Anirudh Date: Fri, 22 Nov 2024 17:40:49 -0500 Subject: [PATCH 2/3] fix: integrate enforce_response_format and add tests --- .../interior_design_assistant/tests_utils.py | 34 +++++++++++++++++++ examples/interior_design_assistant/utils.py | 30 ++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 examples/interior_design_assistant/tests_utils.py diff --git a/examples/interior_design_assistant/tests_utils.py b/examples/interior_design_assistant/tests_utils.py new file mode 100644 index 000000000..33b20ccc5 --- /dev/null +++ b/examples/interior_design_assistant/tests_utils.py @@ -0,0 +1,34 @@ +import pytest +from examples.interior_design_assistant.utils import enforce_response_format + +def test_valid_output(): + output = """ + [ + {"description": "Modern leather sofa with sleek black finish"}, + {"description": "Oak wood coffee table with glass top"}, + {"description": "Elegant brass floor lamp with a white shade"} + ] + """ + result = enforce_response_format(output, 3) + assert result == [ + "Modern leather sofa with sleek black finish", + "Oak wood coffee table with glass top", + "Elegant brass floor lamp with a white shade" + ] + +def test_invalid_json_output(): + output = "Invalid output" + result = enforce_response_format(output, 3) + assert len(result) == 3 + assert all("Alternative suggestion" in desc for desc in result) + +def test_missing_description_field(): + output = """ + [ + {"not_description": "This is not valid"}, + {"not_description": "Neither is this"} + ] + """ + result = enforce_response_format(output, 3) + assert len(result) == 3 + assert all("Alternative suggestion" in desc for desc in result) diff --git a/examples/interior_design_assistant/utils.py b/examples/interior_design_assistant/utils.py index 7defe5f3d..3821e6f48 100644 --- a/examples/interior_design_assistant/utils.py +++ b/examples/interior_design_assistant/utils.py @@ -8,6 +8,36 @@ import mimetypes import uuid +import json + +def enforce_response_format(output, n): + """ + Validate the JSON output from the model. + + Args: + output (str): Raw string output from the model. + n (int): Number of descriptions expected in the output. + + Returns: + list: List of validated alternative descriptions. + """ + try: + parsed_output = json.loads(output.strip()) + # Ensure it is a list and contains valid "description" fields + if isinstance(parsed_output, list): + validated_descriptions = [ + item.get("description", f"Alternative suggestion {i+1}") + for i, item in enumerate(parsed_output) + ] + # If there are fewer descriptions than expected, add fallback suggestions + while len(validated_descriptions) < n: + validated_descriptions.append(f"Alternative suggestion {len(validated_descriptions)+1}") + return validated_descriptions + else: + raise ValueError("Output is not a valid list") + except (json.JSONDecodeError, ValueError): + # Return fallback suggestions for invalid JSON or unexpected structure + return [f"Alternative suggestion {i+1}" for i in range(n)] # TODO: This should move into a common util as will be needed by all apps def data_url_from_image(file_path): From ac9819660c50c3e73b5b63f111059c336c1dc4af Mon Sep 17 00:00:00 2001 From: Anirudh Date: Mon, 25 Nov 2024 17:15:46 -0500 Subject: [PATCH 3/3] fix: enforce response_format for structured JSON outputs --- examples/interior_design_assistant/api.py | 33 +++++++++++++++--- .../interior_design_assistant/tests_utils.py | 34 ------------------- examples/interior_design_assistant/utils.py | 29 ---------------- 3 files changed, 28 insertions(+), 68 deletions(-) delete mode 100644 examples/interior_design_assistant/tests_utils.py diff --git a/examples/interior_design_assistant/api.py b/examples/interior_design_assistant/api.py index a2f3a4728..b2a246511 100644 --- a/examples/interior_design_assistant/api.py +++ b/examples/interior_design_assistant/api.py @@ -10,7 +10,7 @@ import uuid from pathlib import Path from typing import List -from examples.interior_design_assistant.utils import enforce_response_format + import fire from examples.interior_design_assistant.utils import ( @@ -61,6 +61,16 @@ async def list_items(self, file_path: str) -> List[str]: assert ( self.agent_id is not None ), "Agent not initialized, call initialize() first" + + response_format = { + "type": "object", + "properties": { + "description": {"type": "string"}, + "items": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["description", "items"] + } + text = textwrap.dedent( """ Analyze the image to provide a 4 sentence description of the architecture and furniture items present in it. @@ -95,6 +105,7 @@ async def list_items(self, file_path: str) -> List[str]: session_id=resposne.session_id, messages=[message], stream=True, + response_format=response_format, ) result = "" @@ -106,8 +117,8 @@ async def list_items(self, file_path: str) -> List[str]: # print(turn.output_message.content) result = turn.output_message.content - d = json.loads(result.strip()) - return d + return json.loads(result) + async def suggest_alternatives( self, file_path: str, item: str, n: int = 3 @@ -116,6 +127,16 @@ async def suggest_alternatives( Analyze the image using multimodal llm and return possible alternative descriptions for the provided item. """ + response_format = { + "type": "array", + "items": { + "type": "object", + "properties": { + "description": {"type": "string"} + }, + "required": ["description"] + } + } prompt = textwrap.dedent( """ For the given image, your task is to carefully examine the image to provide alternative suggestions for {item}. @@ -154,11 +175,13 @@ async def suggest_alternatives( agent_id=self.agent_id, session_name=uuid.uuid4().hex, ) + generator = self.client.agents.turn.create( agent_id=self.agent_id, session_id=resposne.session_id, messages=[message], stream=True, + response_format=response_format, ) result = "" for chunk in generator: @@ -167,8 +190,8 @@ async def suggest_alternatives( turn = payload.turn result = turn.output_message.content - print(result) - return enforce_response_format(result, n) + print(result) + return [r["description"] for r in json.loads(result)] async def retrieve_images(self, description: str) -> List[ImageMedia]: """ diff --git a/examples/interior_design_assistant/tests_utils.py b/examples/interior_design_assistant/tests_utils.py deleted file mode 100644 index 33b20ccc5..000000000 --- a/examples/interior_design_assistant/tests_utils.py +++ /dev/null @@ -1,34 +0,0 @@ -import pytest -from examples.interior_design_assistant.utils import enforce_response_format - -def test_valid_output(): - output = """ - [ - {"description": "Modern leather sofa with sleek black finish"}, - {"description": "Oak wood coffee table with glass top"}, - {"description": "Elegant brass floor lamp with a white shade"} - ] - """ - result = enforce_response_format(output, 3) - assert result == [ - "Modern leather sofa with sleek black finish", - "Oak wood coffee table with glass top", - "Elegant brass floor lamp with a white shade" - ] - -def test_invalid_json_output(): - output = "Invalid output" - result = enforce_response_format(output, 3) - assert len(result) == 3 - assert all("Alternative suggestion" in desc for desc in result) - -def test_missing_description_field(): - output = """ - [ - {"not_description": "This is not valid"}, - {"not_description": "Neither is this"} - ] - """ - result = enforce_response_format(output, 3) - assert len(result) == 3 - assert all("Alternative suggestion" in desc for desc in result) diff --git a/examples/interior_design_assistant/utils.py b/examples/interior_design_assistant/utils.py index 3821e6f48..05dafd518 100644 --- a/examples/interior_design_assistant/utils.py +++ b/examples/interior_design_assistant/utils.py @@ -10,35 +10,6 @@ import json -def enforce_response_format(output, n): - """ - Validate the JSON output from the model. - - Args: - output (str): Raw string output from the model. - n (int): Number of descriptions expected in the output. - - Returns: - list: List of validated alternative descriptions. - """ - try: - parsed_output = json.loads(output.strip()) - # Ensure it is a list and contains valid "description" fields - if isinstance(parsed_output, list): - validated_descriptions = [ - item.get("description", f"Alternative suggestion {i+1}") - for i, item in enumerate(parsed_output) - ] - # If there are fewer descriptions than expected, add fallback suggestions - while len(validated_descriptions) < n: - validated_descriptions.append(f"Alternative suggestion {len(validated_descriptions)+1}") - return validated_descriptions - else: - raise ValueError("Output is not a valid list") - except (json.JSONDecodeError, ValueError): - # Return fallback suggestions for invalid JSON or unexpected structure - return [f"Alternative suggestion {i+1}" for i in range(n)] - # TODO: This should move into a common util as will be needed by all apps def data_url_from_image(file_path): mime_type, _ = mimetypes.guess_type(file_path)