prompt updates

app/agent/agent_generation.py

@@ -22,12 +22,10 @@ def generate_answer(self, question: str, context: str) -> str:
         LogWriter.info(f"request_id={req_id_cv.get()} ENTRY generate_answer")
         prompt = PromptTemplate(
             template="""Given the question and the context, generate an answer. \n
-                        Use the context to generate the answer. \n
-                        Make sure to answer the question in a friendly, but concise, and informative way. \n
-                        Return the answer as a string. \n
+                        Make sure to answer the question in a friendly and informative way. \n
                         Question: {question} \n
                         Context: {context}""",
-            input_variables=["generation", "context"]
+            input_variables=["question", "context"]
         )
 
         # Chain

app/agent/agent_graph.py

@@ -13,6 +13,12 @@
 
 from app.py_schemas import (MapQuestionToSchemaResponse,
                             CoPilotResponse)
+
+import logging
+from app.log import req_id_cv
+
+logger = logging.getLogger(__name__)
+
 class GraphState(TypedDict):
     """
     Represents the state of the agent graph.
@@ -54,7 +60,9 @@ def route_question(self, state):
         elif state["question_retry_count"] > 2:
             return "apologize"
         state["question_retry_count"] += 1
+        logger.debug_pii(f"request_id={req_id_cv.get()} Routing question: {state['question']}")
         source = step.route_question(state['question'])
+        logger.debug_pii(f"request_id={req_id_cv.get()} Routing question to: {source}")
         if source["datasource"] == "vectorstore":
             return "supportai_lookup"
         elif source["datasource"] == "functions":
@@ -119,7 +127,9 @@ def generate_answer(self, state):
         Run the agent generator.
         """
         step = TigerGraphAgentGenerator(self.llm_provider)
+        logger.debug_pii(f"request_id={req_id_cv.get()} Generating answer for question: {state['question']}")
         answer = step.generate_answer(state['question'], state["context"])
+        logger.debug_pii(f"request_id={req_id_cv.get()} Generated answer: {answer}")
 
         try:
             resp = CoPilotResponse(natural_language_response=answer,
@@ -148,7 +158,7 @@ def check_answer_for_hallucinations(self, state):
         Run the agent hallucination check.
         """
         step = TigerGraphAgentHallucinationCheck(self.llm_provider)
-        hallucinations = step.check_hallucination(state["answer"], state["context"])
+        hallucinations = step.check_hallucination(state["answer"].natural_language_response, state["context"])
         if hallucinations["score"] == "yes":
             return "grounded"
         else:
@@ -159,7 +169,7 @@ def check_answer_for_usefulness(self, state):
         Run the agent usefulness check.
         """
         step = TigerGraphAgentUsefulnessCheck(self.llm_provider)
-        usefulness = step.check_usefulness(state["question"], state["answer"])
+        usefulness = step.check_usefulness(state["question"], state["answer"].natural_language_response)
         if usefulness["score"] == "yes":
             return "useful"
         else:

app/agent/agent_router.py

@@ -1,5 +1,5 @@
 from langchain.prompts import PromptTemplate
-from langchain_core.output_parsers import JsonOutputParser
+from langchain_core.output_parsers import JsonOutputParser, StrOutputParser
 from app.tools.logwriter import LogWriter
 from pyTigerGraph.pyTigerGraph import TigerGraphConnection
 import logging
@@ -31,8 +31,9 @@ def route_question(self, question: str) -> str:
             Keep in mind that some questions about documents such as "how many documents are there?" can be answered by function calls. \n
             The function calls can be used to answer questions about these entities: {v_types} and relationships: {e_types}. \n
             Otherwise, use function calls. Give a binary choice 'functions' or 'vectorstore' based on the question. \n
-            Return the a JSON with a single key 'datasource' and no premable or explaination. \n
-            Question to route: {question}""",
+            Return the a JSON with a single key 'datasource' and no premable or explaination. 
+            Question to route: {question}
+            Remember to only return the JSON document with the key 'datasource' and the value 'functions' or 'vectorstore'""",
             input_variables=["question", "v_types", "e_types"],
         )
 

app/prompts/openai_gpt4/generate_function.txt

@@ -15,6 +15,12 @@ Third Docstring: {doc3}
 Fourth Docstring: {doc4}
 Fifth Docstring: {doc5}
 Sixth Docstring: {doc6}
+Seventh Docstring: {doc7}
+Eighth Docstring: {doc8}
 
+
+Make sure to carefully read the question and the docstrings to determine the correct function to call. 
+Choose the simplest function that will answer the question.
+Only choose one function to call, and do not change the syntax of the function call.
 Follow the output directions below on how to structure your response: 
 {format_instructions} 

app/py_schemas/tool_io_schemas.py

@@ -1,5 +1,5 @@
 from langchain.pydantic_v1 import BaseModel, Field
-from langchain_core.pydantic_v1 import Field
+from typing import Optional
 from langchain_community.graphs.graph_document import (
     Node as BaseNode,
     Relationship as BaseRelationship,
@@ -14,16 +14,16 @@ class MapQuestionToSchemaResponse(BaseModel):
     target_vertex_types: List[str] = Field(
         description="The list of vertices mentioned in the question. If there are no vertices mentioned, then use an empty list."
     )
-    target_vertex_attributes: Dict[str, List[str]] = Field(
+    target_vertex_attributes: Optional[Dict[str, List[str]]] = Field(
         description="The dictionary of vertex attributes mentioned in the question, formated in {'vertex_type_1': ['vertex_attribute_1', 'vertex_attribute_2'], 'vertex_type_2': ['vertex_attribute_1', 'vertex_attribute_2']}"
     )
-    target_vertex_ids: Dict[str, List[str]] = Field(
+    target_vertex_ids: Optional[Dict[str, List[str]]] = Field(
         description="The dictionary of vertex ids mentioned in the question. If there are no vertex ids mentioned, then use an empty dict. formated in {'vertex_type_1': ['vertex_id_1', 'vertex_id_2'], 'vertex_type_2': ['vertex_id_1', 'vertex_id_2']}"
     )
-    target_edge_types: List[str] = Field(
+    target_edge_types: Optional[List[str]] = Field(
         description="The list of edges mentioned in the question"
     )
-    target_edge_attributes: Dict[str, List[str]] = Field(
+    target_edge_attributes: Optional[Dict[str, List[str]]] = Field(
         description="The dictionary of edge attributes mentioned in the question, formated in {'edge_type': ['edge_attribute_1', 'edge_attribute_2']}"
     )
 

app/tg_documents/get_edge_count.json

@@ -1,7 +1,7 @@
 {
     "function_header": "getEdgeCount",
     "description": "Get the number of edges at a graph level, optionally filtered by type.",
-    "docstring": "`getEdgeCount(edgeType: str = '*', sourceVertexType: str = '', targetVertexType: str = '')` → dict\nReturns the number of edges of an edge type.\nThis is a simplified version of getEdgeCountFrom(), to be used when the total number of edges of a given type is needed, regardless which vertex instance they are originated from. See documentation of getEdgeCountFrom above for more details.\nParameters:\nedgeType: The name of the edge type.\nsourceVertexType: The name of the source vertex type.\ntargetVertexType: The name of the target vertex type.\nReturns:\nA dictionary of edge_type: edge_count pairs.",
+    "docstring": "`getEdgeCount(edgeType: str = '*', sourceVertexType: str = '', targetVertexType: str = '')` → dict\nReturns the number of edges of an edge type.\nThis is a simplified version of getEdgeCountFrom(), to be used when the total number of edges of a given type is needed, regardless which vertex instance they are originated from. See documentation of getEdgeCountFrom above for more details.\nParameters:\nedgeType: The name of the edge type.\nsourceVertexType: The name of the source vertex type.\ntargetVertexType: The name of the target vertex type.\nReturns:\nA dictionary of edge_type: edge_count pairs. Execute the function by running `getEdgeCount()` with the appropriate parameters inserted.",
     "param_types": {
         "edgeType": "str",
         "sourceVertexType": "str",

app/tg_documents/get_edge_count_from.json

@@ -1,7 +1,7 @@
 {
     "function_header": "getEdgeCountFrom",
     "description": "Get the number of edges from a source vertex, and optionally to a target vertex.",
-    "docstring": "`getEdgeCountFrom(sourceVertexType: str = '', sourceVertexId: Union[str, int] = None, edgeType: str = '', targetVertexType: str = '', targetVertexId: Union[str, int] = None, where: str = '')` → dict\nReturns the number of edges from a specific vertex.\nParameters:\nsourceVertexType: The name of the source vertex type.\nsourceVertexId: The primary ID value of the source vertex instance.\nedgeType: The name of the edge type.\ntargetVertexType: The name of the target vertex type.\ntargetVertexId: The primary ID value of the target vertex instance.\nwhere: A comma separated list of conditions that are all applied on each edge’s attributes. The conditions are in logical conjunction (i.e. they are 'AND’ed' together).\nReturns:\nA dictionary of edge_type: edge_count pairs.\nUses:\nIf edgeType = '*': edge count of all edge types (no other arguments can be specified in this case).\nIf edgeType is specified only: edge count of the given edge type.\nIf sourceVertexType, edgeType, targetVertexType are specified: edge count of the given edge type between source and target vertex types.\nIf sourceVertexType, sourceVertexId are specified: edge count of all edge types from the given vertex instance.\nIf sourceVertexType, sourceVertexId, edgeType are specified: edge count of all edge types from the given vertex instance.\nIf sourceVertexType, sourceVertexId, edgeType, where are specified: the edge count of the given edge type after filtered by where condition.\nIf targetVertexId is specified, then targetVertexType must also be specified.\nIf targetVertexType is specified, then edgeType must also be specified.",
+    "docstring": "`getEdgeCountFrom(sourceVertexType: str = '', sourceVertexId: Union[str, int] = None, edgeType: str = '', targetVertexType: str = '', targetVertexId: Union[str, int] = None, where: str = '')` → dict\nReturns the number of edges from a specific vertex.\nParameters:\nsourceVertexType: The name of the source vertex type.\nsourceVertexId: The primary ID value of the source vertex instance.\nedgeType: The name of the edge type.\ntargetVertexType: The name of the target vertex type.\ntargetVertexId: The primary ID value of the target vertex instance.\nwhere: A comma separated list of conditions that are all applied on each edge’s attributes. The conditions are in logical conjunction (i.e. they are 'AND’ed' together).\nReturns:\nA dictionary of edge_type: edge_count pairs.\nUses:\nIf edgeType = '*': edge count of all edge types (no other arguments can be specified in this case).\nIf edgeType is specified only: edge count of the given edge type.\nIf sourceVertexType, edgeType, targetVertexType are specified: edge count of the given edge type between source and target vertex types.\nIf sourceVertexType, sourceVertexId are specified: edge count of all edge types from the given vertex instance.\nIf sourceVertexType, sourceVertexId, edgeType are specified: edge count of all edge types from the given vertex instance.\nIf sourceVertexType, sourceVertexId, edgeType, where are specified: the edge count of the given edge type after filtered by where condition.\nIf targetVertexId is specified, then targetVertexType must also be specified.\nIf targetVertexType is specified, then edgeType must also be specified. Execute the function by running `getEdgeCountFrom()` with the appropriate parameters inserted.",
     "param_types": {
         "sourceVertexType": "str",
         "sourceVertexId": "Union[str, int]",

app/tg_documents/get_edge_stats.json

@@ -1,7 +1,7 @@
 {
     "function_header": "getEdgeStats",
     "description": "Retrieves edge attribute statistics",
-    "docstring": "`getEdgeStats(edgeTypes: Union[str, list], skipNA: bool = False)` → dict\nReturns edge attribute statistics.\nParameters:\nedgeTypes: A single edge type name or a list of edges types names or '*' for all edges types.\nskipNA: Skip those edges that do not have attributes or none of their attributes have statistics gathered.\nReturns:\nAttribute statistics of edges; a dictionary of dictionaries.",
+    "docstring": "`getEdgeStats(edgeTypes: Union[str, list], skipNA: bool = False)` → dict\nReturns edge attribute statistics.\nParameters:\nedgeTypes: A single edge type name or a list of edges types names or '*' for all edges types.\nskipNA: Skip those edges that do not have attributes or none of their attributes have statistics gathered.\nReturns:\nAttribute statistics of edges; a dictionary of dictionaries. Execute the function by running `getEdgeStats()` with the appropriate parameters inserted.",
     "param_types": {
         "edgeTypes": "Union[str, list]",
         "skipNA": "bool"

app/tg_documents/get_edges.json

@@ -1,7 +1,7 @@
 {
     "function_header": "getEdges",
     "description": "Return edges from the database that comply with certain conditions",
-    "docstring": "`getEdges(sourceVertexType: str, sourceVertexId: Union[str, int], edgeType: str = '', targetVertexType: str = '', targetVertexId: Union[str, int] = '', where: str = '', limit: Union[int, str] = None, sort: str = '')` → Union[dict, str, pd.DataFrame]\nRetrieves edges of the given edge type originating from a specific source vertex.\nOnly sourceVertexType and sourceVertexId are required. If targetVertexId is specified, then targetVertexType must also be specified. If targetVertexType is specified, then edgeType must also be specified.\nParameters:\nsourceVertexType: The name of the source vertex type.\nsourceVertexId: The primary ID value of the source vertex instance.\nedgeType: The name of the edge type.\ntargetVertexType: The name of the target vertex type.\ntargetVertexId: The primary ID value of the target vertex instance.\nwhere: Comma separated list of conditions that are all applied on each edge’s attributes. The conditions are in logical conjunction (i.e. they are 'AND’ed' together).\nsort: Comma separated list of attributes the results should be sorted by.\nlimit: Maximum number of edge instances to be returned (after sorting).\nReturns:\nThe (selected) details of the (matching) edge instances (sorted, limited) as JSON.",
+    "docstring": "`getEdges(sourceVertexType: str, sourceVertexId: Union[str, int], edgeType: str = '', targetVertexType: str = '', targetVertexId: Union[str, int] = '', where: str = '', limit: Union[int, str] = None, sort: str = '')` → Union[dict, str, pd.DataFrame]\nRetrieves edges of the given edge type originating from a specific source vertex.\nOnly sourceVertexType and sourceVertexId are required. If targetVertexId is specified, then targetVertexType must also be specified. If targetVertexType is specified, then edgeType must also be specified.\nParameters:\nsourceVertexType: The name of the source vertex type.\nsourceVertexId: The primary ID value of the source vertex instance.\nedgeType: The name of the edge type.\ntargetVertexType: The name of the target vertex type.\ntargetVertexId: The primary ID value of the target vertex instance.\nwhere: Comma separated list of conditions that are all applied on each edge’s attributes. The conditions are in logical conjunction (i.e. they are 'AND’ed' together).\nsort: Comma separated list of attributes the results should be sorted by.\nlimit: Maximum number of edge instances to be returned (after sorting).\nReturns:\nThe (selected) details of the (matching) edge instances (sorted, limited) as JSON. Execute the function by running `getEdges()` with the appropriate parameters inserted.",
     "param_types": {
         "sourceVertexType": "str",
         "sourceVertexId": "Union[str, int]",

app/tg_documents/get_vertex_count.json

@@ -1,7 +1,7 @@
 {
     "function_header": "getVertexCount",
     "description": "Get the count of a vertex type, optionally with a where filter",
-    "docstring": "`getVertexCount(vertexType: Union[str, list] = '*', where: str = '')` → Union[int, dict]\nReturns the number of vertices of the specified type.\nParameters:\nvertexType (Union[str, list], optional): The name of the vertex type. If vertexType == '*', then count the instances of all vertex types (where cannot be specified in this case). Defaults to '*'.\nwhere (str, optional): A comma separated list of conditions that are all applied on each vertex’s attributes. The conditions are in logical conjunction (i.e. they are 'AND’ed' together). Defaults to ''.\nReturns:\nA dictionary of <vertex_type>: <vertex_count> pairs if vertexType is a list or '*'.\nAn integer of vertex count if vertexType is a single vertex type.\nUses:\nIf vertexType is specified only: count of the instances of the given vertex type(s).\nIf vertexType and where are specified: count of the instances of the given vertex type after being filtered by where condition(s).",
+    "docstring": "`getVertexCount(vertexType: Union[str, list] = '*', where: str = '')` → Union[int, dict]\nReturns the number of vertices of the specified type.\nParameters:\nvertexType (Union[str, list], optional): The name of the vertex type. If vertexType == '*', then count the instances of all vertex types (where cannot be specified in this case). Defaults to '*'.\nwhere (str, optional): A comma separated list of conditions that are all applied on each vertex’s attributes. The conditions are in logical conjunction (i.e. they are 'AND’ed' together). Defaults to ''.\nReturns:\nA dictionary of <vertex_type>: <vertex_count> pairs if vertexType is a list or '*'.\nAn integer of vertex count if vertexType is a single vertex type.\nUses:\nIf vertexType is specified only: count of the instances of the given vertex type(s).\nIf vertexType and where are specified: count of the instances of the given vertex type after being filtered by where condition(s). Execute the function by running `getVertexCount()` with the appropriate parameters inserted.",
     "param_types": {
         "vertexType": "Union[str, List[str]]",
         "where": "str"

app/tg_documents/get_vertex_stats.json

@@ -1,7 +1,7 @@
 {
     "function_header": "getVertexStats",
     "description": "Get the statistics of vertex attributes.",
-    "docstring": "`getVertexStats(vertexTypes: Union[str, list], skipNA: bool = False) → dict`\nReturns vertex attribute statistics.\nParameters:\nvertexTypes: A single vertex type name or a list of vertex types names or '*' for all vertex types.\nskipNA: Skip those non-applicable vertices that do not have attributes or none of their attributes have statistics gathered.\nReturns:\nA dictionary of various vertex stats for each vertex type specified.",
+    "docstring": "`getVertexStats(vertexTypes: Union[str, list], skipNA: bool = False) → dict`\nReturns vertex attribute statistics.\nParameters:\nvertexTypes: A single vertex type name or a list of vertex types names or '*' for all vertex types.\nskipNA: Skip those non-applicable vertices that do not have attributes or none of their attributes have statistics gathered.\nReturns:\nA dictionary of various vertex stats for each vertex type specified. Execute the function by running `getVertexStats()` with the appropriate parameters inserted.",
     "param_types": {
         "vertexTypes": "Union[str, list]",
         "skipNA": "bool"