diff --git a/app/agent/agent_generation.py b/app/agent/agent_generation.py index 001bfacf..69d911d2 100644 --- a/app/agent/agent_generation.py +++ b/app/agent/agent_generation.py @@ -22,12 +22,10 @@ def generate_answer(self, question: str, context: str) -> str: LogWriter.info(f"request_id={req_id_cv.get()} ENTRY generate_answer") prompt = PromptTemplate( template="""Given the question and the context, generate an answer. \n - Use the context to generate the answer. \n - Make sure to answer the question in a friendly, but concise, and informative way. \n - Return the answer as a string. \n + Make sure to answer the question in a friendly and informative way. \n Question: {question} \n Context: {context}""", - input_variables=["generation", "context"] + input_variables=["question", "context"] ) # Chain diff --git a/app/agent/agent_graph.py b/app/agent/agent_graph.py index 2914b0b5..c1546b84 100644 --- a/app/agent/agent_graph.py +++ b/app/agent/agent_graph.py @@ -13,6 +13,12 @@ from app.py_schemas import (MapQuestionToSchemaResponse, CoPilotResponse) + +import logging +from app.log import req_id_cv + +logger = logging.getLogger(__name__) + class GraphState(TypedDict): """ Represents the state of the agent graph. @@ -54,7 +60,9 @@ def route_question(self, state): elif state["question_retry_count"] > 2: return "apologize" state["question_retry_count"] += 1 + logger.debug_pii(f"request_id={req_id_cv.get()} Routing question: {state['question']}") source = step.route_question(state['question']) + logger.debug_pii(f"request_id={req_id_cv.get()} Routing question to: {source}") if source["datasource"] == "vectorstore": return "supportai_lookup" elif source["datasource"] == "functions": @@ -119,7 +127,9 @@ def generate_answer(self, state): Run the agent generator. """ step = TigerGraphAgentGenerator(self.llm_provider) + logger.debug_pii(f"request_id={req_id_cv.get()} Generating answer for question: {state['question']}") answer = step.generate_answer(state['question'], state["context"]) + logger.debug_pii(f"request_id={req_id_cv.get()} Generated answer: {answer}") try: resp = CoPilotResponse(natural_language_response=answer, @@ -148,7 +158,7 @@ def check_answer_for_hallucinations(self, state): Run the agent hallucination check. """ step = TigerGraphAgentHallucinationCheck(self.llm_provider) - hallucinations = step.check_hallucination(state["answer"], state["context"]) + hallucinations = step.check_hallucination(state["answer"].natural_language_response, state["context"]) if hallucinations["score"] == "yes": return "grounded" else: @@ -159,7 +169,7 @@ def check_answer_for_usefulness(self, state): Run the agent usefulness check. """ step = TigerGraphAgentUsefulnessCheck(self.llm_provider) - usefulness = step.check_usefulness(state["question"], state["answer"]) + usefulness = step.check_usefulness(state["question"], state["answer"].natural_language_response) if usefulness["score"] == "yes": return "useful" else: diff --git a/app/agent/agent_router.py b/app/agent/agent_router.py index 5d9a8b8c..f71569fa 100644 --- a/app/agent/agent_router.py +++ b/app/agent/agent_router.py @@ -1,5 +1,5 @@ from langchain.prompts import PromptTemplate -from langchain_core.output_parsers import JsonOutputParser +from langchain_core.output_parsers import JsonOutputParser, StrOutputParser from app.tools.logwriter import LogWriter from pyTigerGraph.pyTigerGraph import TigerGraphConnection import logging @@ -31,8 +31,9 @@ def route_question(self, question: str) -> str: Keep in mind that some questions about documents such as "how many documents are there?" can be answered by function calls. \n The function calls can be used to answer questions about these entities: {v_types} and relationships: {e_types}. \n Otherwise, use function calls. Give a binary choice 'functions' or 'vectorstore' based on the question. \n - Return the a JSON with a single key 'datasource' and no premable or explaination. \n - Question to route: {question}""", + Return the a JSON with a single key 'datasource' and no premable or explaination. + Question to route: {question} + Remember to only return the JSON document with the key 'datasource' and the value 'functions' or 'vectorstore'""", input_variables=["question", "v_types", "e_types"], ) diff --git a/app/prompts/openai_gpt4/generate_function.txt b/app/prompts/openai_gpt4/generate_function.txt index 0166f449..c3c8cc5d 100644 --- a/app/prompts/openai_gpt4/generate_function.txt +++ b/app/prompts/openai_gpt4/generate_function.txt @@ -15,6 +15,12 @@ Third Docstring: {doc3} Fourth Docstring: {doc4} Fifth Docstring: {doc5} Sixth Docstring: {doc6} +Seventh Docstring: {doc7} +Eighth Docstring: {doc8} + +Make sure to carefully read the question and the docstrings to determine the correct function to call. +Choose the simplest function that will answer the question. +Only choose one function to call, and do not change the syntax of the function call. Follow the output directions below on how to structure your response: {format_instructions} diff --git a/app/py_schemas/tool_io_schemas.py b/app/py_schemas/tool_io_schemas.py index 1a0c8c22..027e6388 100644 --- a/app/py_schemas/tool_io_schemas.py +++ b/app/py_schemas/tool_io_schemas.py @@ -1,5 +1,5 @@ from langchain.pydantic_v1 import BaseModel, Field -from langchain_core.pydantic_v1 import Field +from typing import Optional from langchain_community.graphs.graph_document import ( Node as BaseNode, Relationship as BaseRelationship, @@ -14,16 +14,16 @@ class MapQuestionToSchemaResponse(BaseModel): target_vertex_types: List[str] = Field( description="The list of vertices mentioned in the question. If there are no vertices mentioned, then use an empty list." ) - target_vertex_attributes: Dict[str, List[str]] = Field( + target_vertex_attributes: Optional[Dict[str, List[str]]] = Field( description="The dictionary of vertex attributes mentioned in the question, formated in {'vertex_type_1': ['vertex_attribute_1', 'vertex_attribute_2'], 'vertex_type_2': ['vertex_attribute_1', 'vertex_attribute_2']}" ) - target_vertex_ids: Dict[str, List[str]] = Field( + target_vertex_ids: Optional[Dict[str, List[str]]] = Field( description="The dictionary of vertex ids mentioned in the question. If there are no vertex ids mentioned, then use an empty dict. formated in {'vertex_type_1': ['vertex_id_1', 'vertex_id_2'], 'vertex_type_2': ['vertex_id_1', 'vertex_id_2']}" ) - target_edge_types: List[str] = Field( + target_edge_types: Optional[List[str]] = Field( description="The list of edges mentioned in the question" ) - target_edge_attributes: Dict[str, List[str]] = Field( + target_edge_attributes: Optional[Dict[str, List[str]]] = Field( description="The dictionary of edge attributes mentioned in the question, formated in {'edge_type': ['edge_attribute_1', 'edge_attribute_2']}" ) diff --git a/app/tg_documents/get_edge_count.json b/app/tg_documents/get_edge_count.json index c3039dd0..d1e86a82 100644 --- a/app/tg_documents/get_edge_count.json +++ b/app/tg_documents/get_edge_count.json @@ -1,7 +1,7 @@ { "function_header": "getEdgeCount", "description": "Get the number of edges at a graph level, optionally filtered by type.", - "docstring": "`getEdgeCount(edgeType: str = '*', sourceVertexType: str = '', targetVertexType: str = '')` → dict\nReturns the number of edges of an edge type.\nThis is a simplified version of getEdgeCountFrom(), to be used when the total number of edges of a given type is needed, regardless which vertex instance they are originated from. See documentation of getEdgeCountFrom above for more details.\nParameters:\nedgeType: The name of the edge type.\nsourceVertexType: The name of the source vertex type.\ntargetVertexType: The name of the target vertex type.\nReturns:\nA dictionary of edge_type: edge_count pairs.", + "docstring": "`getEdgeCount(edgeType: str = '*', sourceVertexType: str = '', targetVertexType: str = '')` → dict\nReturns the number of edges of an edge type.\nThis is a simplified version of getEdgeCountFrom(), to be used when the total number of edges of a given type is needed, regardless which vertex instance they are originated from. See documentation of getEdgeCountFrom above for more details.\nParameters:\nedgeType: The name of the edge type.\nsourceVertexType: The name of the source vertex type.\ntargetVertexType: The name of the target vertex type.\nReturns:\nA dictionary of edge_type: edge_count pairs. Execute the function by running `getEdgeCount()` with the appropriate parameters inserted.", "param_types": { "edgeType": "str", "sourceVertexType": "str", diff --git a/app/tg_documents/get_edge_count_from.json b/app/tg_documents/get_edge_count_from.json index 78416acb..8edec929 100644 --- a/app/tg_documents/get_edge_count_from.json +++ b/app/tg_documents/get_edge_count_from.json @@ -1,7 +1,7 @@ { "function_header": "getEdgeCountFrom", "description": "Get the number of edges from a source vertex, and optionally to a target vertex.", - "docstring": "`getEdgeCountFrom(sourceVertexType: str = '', sourceVertexId: Union[str, int] = None, edgeType: str = '', targetVertexType: str = '', targetVertexId: Union[str, int] = None, where: str = '')` → dict\nReturns the number of edges from a specific vertex.\nParameters:\nsourceVertexType: The name of the source vertex type.\nsourceVertexId: The primary ID value of the source vertex instance.\nedgeType: The name of the edge type.\ntargetVertexType: The name of the target vertex type.\ntargetVertexId: The primary ID value of the target vertex instance.\nwhere: A comma separated list of conditions that are all applied on each edge’s attributes. The conditions are in logical conjunction (i.e. they are 'AND’ed' together).\nReturns:\nA dictionary of edge_type: edge_count pairs.\nUses:\nIf edgeType = '*': edge count of all edge types (no other arguments can be specified in this case).\nIf edgeType is specified only: edge count of the given edge type.\nIf sourceVertexType, edgeType, targetVertexType are specified: edge count of the given edge type between source and target vertex types.\nIf sourceVertexType, sourceVertexId are specified: edge count of all edge types from the given vertex instance.\nIf sourceVertexType, sourceVertexId, edgeType are specified: edge count of all edge types from the given vertex instance.\nIf sourceVertexType, sourceVertexId, edgeType, where are specified: the edge count of the given edge type after filtered by where condition.\nIf targetVertexId is specified, then targetVertexType must also be specified.\nIf targetVertexType is specified, then edgeType must also be specified.", + "docstring": "`getEdgeCountFrom(sourceVertexType: str = '', sourceVertexId: Union[str, int] = None, edgeType: str = '', targetVertexType: str = '', targetVertexId: Union[str, int] = None, where: str = '')` → dict\nReturns the number of edges from a specific vertex.\nParameters:\nsourceVertexType: The name of the source vertex type.\nsourceVertexId: The primary ID value of the source vertex instance.\nedgeType: The name of the edge type.\ntargetVertexType: The name of the target vertex type.\ntargetVertexId: The primary ID value of the target vertex instance.\nwhere: A comma separated list of conditions that are all applied on each edge’s attributes. The conditions are in logical conjunction (i.e. they are 'AND’ed' together).\nReturns:\nA dictionary of edge_type: edge_count pairs.\nUses:\nIf edgeType = '*': edge count of all edge types (no other arguments can be specified in this case).\nIf edgeType is specified only: edge count of the given edge type.\nIf sourceVertexType, edgeType, targetVertexType are specified: edge count of the given edge type between source and target vertex types.\nIf sourceVertexType, sourceVertexId are specified: edge count of all edge types from the given vertex instance.\nIf sourceVertexType, sourceVertexId, edgeType are specified: edge count of all edge types from the given vertex instance.\nIf sourceVertexType, sourceVertexId, edgeType, where are specified: the edge count of the given edge type after filtered by where condition.\nIf targetVertexId is specified, then targetVertexType must also be specified.\nIf targetVertexType is specified, then edgeType must also be specified. Execute the function by running `getEdgeCountFrom()` with the appropriate parameters inserted.", "param_types": { "sourceVertexType": "str", "sourceVertexId": "Union[str, int]", diff --git a/app/tg_documents/get_edge_stats.json b/app/tg_documents/get_edge_stats.json index 69a26ee1..264a17bb 100644 --- a/app/tg_documents/get_edge_stats.json +++ b/app/tg_documents/get_edge_stats.json @@ -1,7 +1,7 @@ { "function_header": "getEdgeStats", "description": "Retrieves edge attribute statistics", - "docstring": "`getEdgeStats(edgeTypes: Union[str, list], skipNA: bool = False)` → dict\nReturns edge attribute statistics.\nParameters:\nedgeTypes: A single edge type name or a list of edges types names or '*' for all edges types.\nskipNA: Skip those edges that do not have attributes or none of their attributes have statistics gathered.\nReturns:\nAttribute statistics of edges; a dictionary of dictionaries.", + "docstring": "`getEdgeStats(edgeTypes: Union[str, list], skipNA: bool = False)` → dict\nReturns edge attribute statistics.\nParameters:\nedgeTypes: A single edge type name or a list of edges types names or '*' for all edges types.\nskipNA: Skip those edges that do not have attributes or none of their attributes have statistics gathered.\nReturns:\nAttribute statistics of edges; a dictionary of dictionaries. Execute the function by running `getEdgeStats()` with the appropriate parameters inserted.", "param_types": { "edgeTypes": "Union[str, list]", "skipNA": "bool" diff --git a/app/tg_documents/get_edges.json b/app/tg_documents/get_edges.json index 293d9de5..ed128ac5 100644 --- a/app/tg_documents/get_edges.json +++ b/app/tg_documents/get_edges.json @@ -1,7 +1,7 @@ { "function_header": "getEdges", "description": "Return edges from the database that comply with certain conditions", - "docstring": "`getEdges(sourceVertexType: str, sourceVertexId: Union[str, int], edgeType: str = '', targetVertexType: str = '', targetVertexId: Union[str, int] = '', where: str = '', limit: Union[int, str] = None, sort: str = '')` → Union[dict, str, pd.DataFrame]\nRetrieves edges of the given edge type originating from a specific source vertex.\nOnly sourceVertexType and sourceVertexId are required. If targetVertexId is specified, then targetVertexType must also be specified. If targetVertexType is specified, then edgeType must also be specified.\nParameters:\nsourceVertexType: The name of the source vertex type.\nsourceVertexId: The primary ID value of the source vertex instance.\nedgeType: The name of the edge type.\ntargetVertexType: The name of the target vertex type.\ntargetVertexId: The primary ID value of the target vertex instance.\nwhere: Comma separated list of conditions that are all applied on each edge’s attributes. The conditions are in logical conjunction (i.e. they are 'AND’ed' together).\nsort: Comma separated list of attributes the results should be sorted by.\nlimit: Maximum number of edge instances to be returned (after sorting).\nReturns:\nThe (selected) details of the (matching) edge instances (sorted, limited) as JSON.", + "docstring": "`getEdges(sourceVertexType: str, sourceVertexId: Union[str, int], edgeType: str = '', targetVertexType: str = '', targetVertexId: Union[str, int] = '', where: str = '', limit: Union[int, str] = None, sort: str = '')` → Union[dict, str, pd.DataFrame]\nRetrieves edges of the given edge type originating from a specific source vertex.\nOnly sourceVertexType and sourceVertexId are required. If targetVertexId is specified, then targetVertexType must also be specified. If targetVertexType is specified, then edgeType must also be specified.\nParameters:\nsourceVertexType: The name of the source vertex type.\nsourceVertexId: The primary ID value of the source vertex instance.\nedgeType: The name of the edge type.\ntargetVertexType: The name of the target vertex type.\ntargetVertexId: The primary ID value of the target vertex instance.\nwhere: Comma separated list of conditions that are all applied on each edge’s attributes. The conditions are in logical conjunction (i.e. they are 'AND’ed' together).\nsort: Comma separated list of attributes the results should be sorted by.\nlimit: Maximum number of edge instances to be returned (after sorting).\nReturns:\nThe (selected) details of the (matching) edge instances (sorted, limited) as JSON. Execute the function by running `getEdges()` with the appropriate parameters inserted.", "param_types": { "sourceVertexType": "str", "sourceVertexId": "Union[str, int]", diff --git a/app/tg_documents/get_vertex_count.json b/app/tg_documents/get_vertex_count.json index e3c67710..afa1cc26 100644 --- a/app/tg_documents/get_vertex_count.json +++ b/app/tg_documents/get_vertex_count.json @@ -1,7 +1,7 @@ { "function_header": "getVertexCount", "description": "Get the count of a vertex type, optionally with a where filter", - "docstring": "`getVertexCount(vertexType: Union[str, list] = '*', where: str = '')` → Union[int, dict]\nReturns the number of vertices of the specified type.\nParameters:\nvertexType (Union[str, list], optional): The name of the vertex type. If vertexType == '*', then count the instances of all vertex types (where cannot be specified in this case). Defaults to '*'.\nwhere (str, optional): A comma separated list of conditions that are all applied on each vertex’s attributes. The conditions are in logical conjunction (i.e. they are 'AND’ed' together). Defaults to ''.\nReturns:\nA dictionary of : pairs if vertexType is a list or '*'.\nAn integer of vertex count if vertexType is a single vertex type.\nUses:\nIf vertexType is specified only: count of the instances of the given vertex type(s).\nIf vertexType and where are specified: count of the instances of the given vertex type after being filtered by where condition(s).", + "docstring": "`getVertexCount(vertexType: Union[str, list] = '*', where: str = '')` → Union[int, dict]\nReturns the number of vertices of the specified type.\nParameters:\nvertexType (Union[str, list], optional): The name of the vertex type. If vertexType == '*', then count the instances of all vertex types (where cannot be specified in this case). Defaults to '*'.\nwhere (str, optional): A comma separated list of conditions that are all applied on each vertex’s attributes. The conditions are in logical conjunction (i.e. they are 'AND’ed' together). Defaults to ''.\nReturns:\nA dictionary of : pairs if vertexType is a list or '*'.\nAn integer of vertex count if vertexType is a single vertex type.\nUses:\nIf vertexType is specified only: count of the instances of the given vertex type(s).\nIf vertexType and where are specified: count of the instances of the given vertex type after being filtered by where condition(s). Execute the function by running `getVertexCount()` with the appropriate parameters inserted.", "param_types": { "vertexType": "Union[str, List[str]]", "where": "str" diff --git a/app/tg_documents/get_vertex_stats.json b/app/tg_documents/get_vertex_stats.json index 06e4922a..408354ff 100644 --- a/app/tg_documents/get_vertex_stats.json +++ b/app/tg_documents/get_vertex_stats.json @@ -1,7 +1,7 @@ { "function_header": "getVertexStats", "description": "Get the statistics of vertex attributes.", - "docstring": "`getVertexStats(vertexTypes: Union[str, list], skipNA: bool = False) → dict`\nReturns vertex attribute statistics.\nParameters:\nvertexTypes: A single vertex type name or a list of vertex types names or '*' for all vertex types.\nskipNA: Skip those non-applicable vertices that do not have attributes or none of their attributes have statistics gathered.\nReturns:\nA dictionary of various vertex stats for each vertex type specified.", + "docstring": "`getVertexStats(vertexTypes: Union[str, list], skipNA: bool = False) → dict`\nReturns vertex attribute statistics.\nParameters:\nvertexTypes: A single vertex type name or a list of vertex types names or '*' for all vertex types.\nskipNA: Skip those non-applicable vertices that do not have attributes or none of their attributes have statistics gathered.\nReturns:\nA dictionary of various vertex stats for each vertex type specified. Execute the function by running `getVertexStats()` with the appropriate parameters inserted.", "param_types": { "vertexTypes": "Union[str, list]", "skipNA": "bool" diff --git a/app/tg_documents/get_vertices.json b/app/tg_documents/get_vertices.json index 270029da..830d53e2 100644 --- a/app/tg_documents/get_vertices.json +++ b/app/tg_documents/get_vertices.json @@ -1,7 +1,7 @@ { "function_header": "getVertices", "description": "Get a sample of vertices", - "docstring": "`getVertices(vertexType: str, where: str = '', limit: Union[int, str] = None, sort: str = '')` → dict\nRetrieves vertices of the given vertex type.\nNote: The primary ID of a vertex instance is NOT an attribute, thus cannot be used in select, where or sort parameters (unless the WITH primary_id_as_attribute clause was used when the vertex type was created).\nUse getVerticesById() if you need to retrieve vertices by their primary ID.\nParameters:\nvertexType: The name of the vertex type.\nwhere: Comma separated list of conditions that are all applied on each vertex' attributes. The conditions are in logical conjunction (i.e. they are 'AND’ed' together).\nsort: Comma separated list of attributes the results should be sorted by. Add '-' in front of the attribute to sort in descending order. Must be used with limit.\nlimit: Maximum number of vertex instances to be returned (after sorting). Must be used with sort.\nReturns:\nThe (selected) details of the (matching) vertex instances (sorted, limited) as dictionary, JSON or pandas DataFrame.", + "docstring": "`getVertices(vertexType: str, where: str = '', limit: Union[int, str] = None, sort: str = '')` → dict\nRetrieves vertices of the given vertex type.\nNote: The primary ID of a vertex instance is NOT an attribute, thus cannot be used in select, where or sort parameters (unless the WITH primary_id_as_attribute clause was used when the vertex type was created).\nUse getVerticesById() if you need to retrieve vertices by their primary ID.\nParameters:\nvertexType: The name of the vertex type.\nwhere: Comma separated list of conditions that are all applied on each vertex' attributes. The conditions are in logical conjunction (i.e. they are 'AND’ed' together).\nsort: Comma separated list of attributes the results should be sorted by. Add '-' in front of the attribute to sort in descending order. Must be used with limit.\nlimit: Maximum number of vertex instances to be returned (after sorting). Must be used with sort.\nReturns:\nThe (selected) details of the (matching) vertex instances (sorted, limited) as dictionary, JSON or pandas DataFrame. Execute the function by running `getVertices()` with the appropriate parameters inserted.", "param_types": { "vertexType": "str", "where": "str", diff --git a/app/tg_documents/get_vertices_by_id.json b/app/tg_documents/get_vertices_by_id.json index e40b6637..1f777ada 100644 --- a/app/tg_documents/get_vertices_by_id.json +++ b/app/tg_documents/get_vertices_by_id.json @@ -1,7 +1,7 @@ { "function_header": "getVerticesById", "description": "Get vertex information by vertex ID", - "docstring": "`getVerticesById(vertexType: str, vertexIds: Union[int, str, list])` → Union[list, str, pd.DataFrame]\nRetrieves vertices of the given vertex type, identified by their ID.\nParameters:\nvertexType: The name of the vertex type.\nvertexIds: A single vertex ID or a list of vertex IDs.\nReturns:\nThe (selected) details of the (matching) vertex instances as JSON.", + "docstring": "`getVerticesById(vertexType: str, vertexIds: Union[int, str, list])` → Union[list, str, pd.DataFrame]\nRetrieves vertices of the given vertex type, identified by their ID.\nParameters:\nvertexType: The name of the vertex type.\nvertexIds: A single vertex ID or a list of vertex IDs.\nReturns:\nThe (selected) details of the (matching) vertex instances as JSON. Execute the function by running `getVerticesById()` with the appropriate parameters inserted.", "param_types": { "vertexType": "str", "vertexIds": "Union[int, str, List[Union[int, str]]"