Gml 1824 regression test with cypher support and no hallucination and usefulness check

common/prompts/aws_bedrock_claude3haiku/map_question_to_schema.txt

@@ -10,4 +10,5 @@ Don't generate target_vertex_ids if there is no the term 'id' explicitly mention
 
 Respond in JSON (and only JSON). Follow the format instructions below:
 {format_instructions}
-QUESTION: {question}
+question: {question}
+conversation: {conversation}

common/prompts/aws_bedrock_titan/map_question_to_schema.txt

@@ -15,4 +15,5 @@ Response: What is the schema?
 Example: How many transactions are there?
 Response: How many TRANSACTION Edges are there?
 {format_instructions}
-QUESTION: {question}
+question: {question}
+conversation: {conversation}

common/prompts/azure_open_ai_gpt35_turbo_instruct/map_question_to_schema.txt

@@ -12,4 +12,6 @@ Generate the complete question with the appropriate replacements. Keep the case
 Format your response following the directions below.
 Don't generate target_vertex_ids if there is no the term 'id' explicitly mentioned in the question.
 
-{format_instructions}
+{format_instructions}
+question: {question}
+conversation: {conversation}

common/prompts/gcp_vertexai_palm/map_question_to_schema.txt

@@ -10,4 +10,5 @@ Generate the complete question with the appropriate replacements. Keep the case
 Don't generate target_vertex_ids if there is no the term 'id' explicitly mentioned in the question.
 
 {format_instructions}
-QUESTION: {question}
+question: {question}
+conversation: {conversation}

common/prompts/llama_70b/map_question_to_schema.txt

@@ -10,4 +10,6 @@ Generate the complete question with the appropriate replacements. Keep the case
 Don't generate target_vertex_ids if there is no the term 'id' explicitly mentioned in the question.
 
 {format_instructions}
-QUESTION: {question}
+question: {question}
+conversation: {conversation}
+

common/prompts/openai_gpt4/map_question_to_schema.txt

@@ -10,5 +10,6 @@ Generate the complete question with the appropriate replacements. Keep the case
 Don't generate target_vertex_ids if there is no the term 'id' explicitly mentioned in the question.
 
 {format_instructions}
-QUESTION: {question}
+question: {question}
+conversation: {conversation}
 

copilot/app/agent/agent.py

@@ -1,4 +1,5 @@
 import logging
+import json
 import time
 from typing import Dict, List
 
@@ -90,7 +91,7 @@ def __init__(
             self.embedding_store,
             self.mq2s,
             self.gen_func,
-            cypher_gen_tool=self.cypher_tool,
+            # cypher_gen_tool=self.cypher_tool,
             q=self.q,
             supportai_retriever=supportai_retriever
         ).create_graph()
@@ -119,20 +120,28 @@ def question_for_agent(
 
             input_data = {}
             input_data["input"] = question
-            logger.info(f"conversation: {conversation}")
 
             if conversation is not None:
                 input_data["conversation"] = [
                     {"query": chat["query"], "response": chat["response"]}
                     for chat in conversation
                 ]
+
             else:
                 # Handle the case where conversation is None or empty
                 input_data["conversation"] = []
-            logger.info(f"input_data: {input_data}")
 
-            for output in self.agent.stream({"question": str(input_data)}):
+            # Validate and convert input_data to JSON string
+            try:
+                input_data_str = json.dumps(input_data)
+            except (TypeError, ValueError) as e:
+                logger.error(f"Failed to serialize input_data to JSON: {e}")
+                raise ValueError("Invalid input data format. Unable to convert to JSON.")
+
+            for output in self.agent.stream({"question": input_data["input"], "conversation": input_data["conversation"]}):
+
                 for key, value in output.items():
+                    logger.info(f"testing steps {key}: {value}")
                     LogWriter.info(f"request_id={req_id_cv.get()} executed node {key}")
 
             LogWriter.info(f"request_id={req_id_cv.get()} EXIT question_for_agent")

copilot/app/agent/agent_generation.py

@@ -1,4 +1,3 @@
-
 import logging
 from langchain.prompts import PromptTemplate
 from langchain_core.output_parsers import PydanticOutputParser
@@ -9,7 +8,7 @@
 logger = logging.getLogger(__name__)
 
 class CoPilotAnswerOutput(BaseModel):
-    generated_answer: str = Field(description="The generated answer to the question. Make sure maintain a professional tone and keep the answer consice.")
+    generated_answer: str = Field(description="The generated answer to the question. Make sure maintain a professional tone.")
     citation: list[str] = Field(description="The citation for the answer. List the information used.")
 
 class TigerGraphAgentGenerator:
@@ -29,19 +28,28 @@ def generate_answer(self, question: str, context: str) -> str:
         answer_parser = PydanticOutputParser(pydantic_object=CoPilotAnswerOutput)
 
         prompt = PromptTemplate(
-            template="""Given the question and the context, generate an answer. \n
-                        Make sure to answer the question in a friendly and informative way. \n
+            template="""Given the answer context in JSON format, rephrase it to answer the question. \n
+                        Use only the provided information in context without adding any reasoning or additional logic. \n
+                        Make sure all information in the answer are covered in the generated answer.\n
                         Question: {question} \n
-                        Context: {context}
+                        Answer: {context} \n
                         Format: {format_instructions}""",
             input_variables=["question", "context"],
             partial_variables={
                 "format_instructions": answer_parser.get_format_instructions()
             }
         )
 
+        full_prompt = prompt.format(
+            question=question,
+            context=context,
+            format_instructions=answer_parser.get_format_instructions()
+        )
+
         # Chain
         rag_chain = prompt | self.llm.model | answer_parser
-        generation = rag_chain.invoke({"context": context, "question": question})
+        generation = rag_chain.invoke({"question": question, "context": context})
+
         LogWriter.info(f"request_id={req_id_cv.get()} EXIT generate_answer")
+
         return generation

copilot/app/agent/agent_graph.py

@@ -1,6 +1,6 @@
 import json
 import logging
-from typing import Optional
+from typing import Dict, List, Optional
 
 from agent.agent_generation import TigerGraphAgentGenerator
 from agent.agent_hallucination_check import TigerGraphAgentHallucinationCheck
@@ -28,6 +28,7 @@ class GraphState(TypedDict):
     """
 
     question: str
+    conversation: Optional[List[Dict[str, str]]]
     generation: str
     context: str
     answer: Optional[CoPilotResponse]
@@ -123,7 +124,7 @@ def map_question_to_schema(self, state):
         """
         self.emit_progress("Mapping your question to the graph's schema")
         try:
-            step = self.mq2s._run(state["question"])
+            step = self.mq2s._run(state["question"], state["conversation"])
             state["schema_mapping"] = step
             return state
         except MapQuestionToSchemaException as e:
@@ -300,7 +301,14 @@ def generate_answer(self, state):
                 state["question"], state["context"]["result"]["@@final_retrieval"]
             )
         elif state["lookup_source"] == "inquiryai":
-            answer = step.generate_answer(state["question"], state["context"]["result"])
+            try:
+                context_data_str = json.dumps(state["context"]["result"])
+            except (TypeError, ValueError) as e:
+                logger.error(f"Failed to serialize context to JSON: {e}")
+                raise ValueError("Invalid context data format. Unable to convert to JSON.")
+
+            answer = step.generate_answer(state["question"], context_data_str)
+
         elif state["lookup_source"] == "cypher":
             answer = step.generate_answer(state["question"], state["context"]["answer"])
         logger.debug_pii(
@@ -312,6 +320,7 @@ def generate_answer(self, state):
 
             citations = [re.sub(r"_chunk_\d+", "", x) for x in answer.citation]
             state["context"]["reasoning"] = list(set(citations))
+
         try:
             resp = CoPilotResponse(
                 natural_language_response=answer.generated_answer,
@@ -336,36 +345,57 @@ def rewrite_question(self, state):
         """
         self.emit_progress("Rephrasing the question")
         step = TigerGraphAgentRewriter(self.llm_provider)
-        state["question"] = step.rewrite_question(state["question"])
+        question_str = state["question"]
+        state["question"] = step.rewrite_question(question_str)
         return state
 
+    # remove halucinaton check, always return grounded
     def check_answer_for_hallucinations(self, state):
         """
         Run the agent hallucination check.
         """
-        self.emit_progress("Checking the response is relevant")
-        step = TigerGraphAgentHallucinationCheck(self.llm_provider)
-        hallucinations = step.check_hallucination(
-            state["answer"].natural_language_response, state["context"]
-        )
-        if hallucinations.score == "yes":
-            self.emit_progress(DONE)
-            return "grounded"
-        else:
-            return "hallucination"
-
+        # self.emit_progress("Checking the response is relevant")
+        # step = TigerGraphAgentHallucinationCheck(self.llm_provider)
+
+        # try:
+        #     context_data_str = json.dumps(state["context"]["result"])
+        #     # logger.info(f"context_data_str: {context_data_str}")
+        # except (TypeError, ValueError) as e:
+        #     logger.error(f"Failed to serialize context to JSON: {e}")
+        #     raise ValueError("Invalid context data format. Unable to convert to JSON.")
+        # hallucinations = step.check_hallucination(
+        #     state["answer"].natural_language_response, context_data_str
+        # )
+        # logger.info(f"hallucination checker")
+        # logger.info(f"answer: {state['answer'].natural_language_response}")
+        # logger.info(f"context: {context_data_str}")
+        # logger.info(f"if grounded: {hallucinations}")
+        # if hallucinations.score == "yes":
+        #     self.emit_progress(DONE)
+        #     return "grounded"
+        # else:
+        #     return "hallucination"
+        return "grounded"
+
+    # remove usefulness check, always return useful
     def check_answer_for_usefulness(self, state):
         """
         Run the agent usefulness check.
         """
-        step = TigerGraphAgentUsefulnessCheck(self.llm_provider)
-        usefulness = step.check_usefulness(
-            state["question"], state["answer"].natural_language_response
-        )
-        if usefulness.score == "yes":
-            return "useful"
-        else:
-            return "not_useful"
+        # step = TigerGraphAgentUsefulnessCheck(self.llm_provider)
+
+        # usefulness = step.check_usefulness(
+        #     state["question"], state["answer"].natural_language_response
+        # )
+        # logger.info(f"usefulness checker")
+        # logger.info(f"question: {state['question']}")
+        # logger.info(f"answer: {state['answer'].natural_language_response}")
+        # logger.info(f"if useful: {usefulness}")
+        # if usefulness.score == "yes":
+        #     return "useful"
+        # else:
+        #     return "not_useful"
+        return "useful"
 
     def check_answer_for_usefulness_and_hallucinations(self, state):
         """
@@ -428,6 +458,7 @@ def create_graph(self):
                 self.check_state_for_generation_error,
                 {"error": "apologize", "success": "generate_answer"},
             )
+            # remove hallucination and usefulness check
             if self.supportai_enabled:
                 self.workflow.add_conditional_edges(
                     "generate_answer",
@@ -457,30 +488,33 @@ def create_graph(self):
                 self.check_state_for_generation_error,
                 {"error": "rewrite_question", "success": "generate_answer"},
             )
-            if self.supportai_enabled:
-                self.workflow.add_conditional_edges(
-                    "generate_answer",
-                    self.check_answer_for_usefulness_and_hallucinations,
-                    {
-                        "hallucination": "rewrite_question",
-                        "grounded": END,
-                        "not_useful": "rewrite_question",
-                        "inquiryai_not_useful": "supportai",
-                        "supportai_not_useful": "map_question_to_schema",
-                    },
-                )
-            else:
-                self.workflow.add_conditional_edges(
-                    "generate_answer",
-                    self.check_answer_for_usefulness_and_hallucinations,
-                    {
-                        "hallucination": "rewrite_question",
-                        "grounded": END,
-                        "not_useful": "rewrite_question",
-                        "inquiryai_not_useful": "apologize",
-                        "supportai_not_useful": "map_question_to_schema",
-                    },
-                )
+
+        if self.supportai_enabled:
+            self.workflow.add_conditional_edges(
+                "generate_answer", 
+                # alwasy return grounded
+                self.check_answer_for_usefulness_and_hallucinations,
+                {
+                    "hallucination": "rewrite_question",
+                    "grounded": END,
+                    "not_useful": "rewrite_question",
+                    "inquiryai_not_useful": "supportai",
+                    "supportai_not_useful": "map_question_to_schema",
+                },
+            )
+        else:
+            self.workflow.add_conditional_edges(
+                "generate_answer", 
+                # always return grounded
+                self.check_answer_for_usefulness_and_hallucinations,
+                {
+                    "hallucination": "rewrite_question",
+                    "grounded": END,
+                    "not_useful": "rewrite_question",
+                    "inquiryai_not_useful": "apologize",
+                    "supportai_not_useful": "map_question_to_schema",
+                },
+            )
 
         if self.supportai_enabled:
             self.workflow.add_conditional_edges(

copilot/app/agent/agent_hallucination_check.py

@@ -27,13 +27,14 @@ def check_hallucination(self, generation: str, context: str) -> dict:
         hallucination_parser = PydanticOutputParser(pydantic_object=HallucinationCheckResponse)        
 
         prompt = PromptTemplate(
-            template="""You are a grader assessing whether an answer is grounded in / supported by a set of facts. \n 
+            template="""You are a grader assessing whether an answer is grounded in / supported by a set of facts. \n
             Here are the facts:
             \n ------- \n
             {context} 
             \n ------- \n
             Here is the answer: {generation}
-            Give a binary score 'yes' or 'no' score to indicate whether the answer is grounded in / supported by a set of facts. \n
+            Provide a binary score 'yes' or 'no' score to indicate whether the answer is grounded in / supported by a set of facts.
+            The score should be 'yes' if the information in the answer is found in the context. otherwise, the score should be 'no'. 
             Provide the binary score as a JSON with a single key 'score' and no preamble or explanation.
             Format: {format_instructions}""",
             input_variables=["generation", "context"],

copilot/app/agent/agent_usefulness_check.py

@@ -27,13 +27,14 @@ def check_usefulness(self, question: str, answer: str) -> dict:
         usefulness_parser = PydanticOutputParser(pydantic_object=UsefulnessCheckResponse)
 
         prompt = PromptTemplate(
-            template="""You are a grader assessing whether an answer is useful to resolve a question. \n 
+            template="""You are a grader assessing whether the generated answer is useful to resolve the question. \n 
             Here is the answer:
             \n ------- \n
             {generation} 
             \n ------- \n
             Here is the question: {question}
-            Give a binary score 'yes' or 'no' to indicate whether the answer is useful to resolve a question. \n
+            Give a binary score 'yes' or 'no' to indicate whether the answer is useful to resolve a question.
+            An answer is considered useful if the answer mostly addresses the question reasonably.
             Provide the binary score as a JSON with a single key 'score' and no preamble or explanation.
             Answer in the format: {format_instructions}""",
             input_variables=["generation", "question"],

copilot/app/tools/generate_function.py

@@ -195,7 +195,6 @@ def _run(
         generated = chain.apply(inputs)[0]["text"]
         logger.debug(f"request_id={req_id_cv.get()} generated function")
         generated = func_parser.invoke(generated)
-        # LogWriter.info(f"generated_function: {generated}")
 
         try:
             parsed_func = validate_function_call(