what is the problem

[amin-kh96]

this is a part of my code.
first, can I use Ollama LLM with ragas? but keep in mind that my LLM runs locally on my computer to avoid API
calls.
so if i want to use it, should I implement it in this way or not?

class OllamaRagasLLM(BaseRagasLLM):
    """Custom Ragas LLM class using Mistral via Ollama."""
    
    def __init__(self, run_config=None, **kwargs):
        # Initialize the parent class with run_config
        if run_config is None:
            # If no run_config is provided, you can define a default one
            run_config = {"some_config_key": "default_value"}  # Example, replace with actual config
        super().__init__(run_config=run_config, **kwargs)
    
    async def _call(self, prompt: str) -> str:
        response = ollama.chat(model="mistral", messages=[{"role": "user", "content": prompt}])
        return response['message']['content']

    async def generate_text(self, prompt, **kwargs) -> LLMResult:
        text = await self._call(prompt)
        return LLMResult(generations=[[Generation(text=text)]])

    async def agenerate_text(self, prompt, **kwargs) -> LLMResult:
        text = await self._call(prompt)
        return LLMResult(generations=[[Generation(text=text)]])

run_config = {"some_config_key": "default_value"}
# Assuming your 'evaluate' function requires 'run_config' to be passed
async def run():
    # Use the custom LLM class with the provided run_config
    llm = OllamaRagasLLM(run_config=run_config)
#custom_llm = CustomRagasLLM(api_key=None, embedding_dataset=embedding_dataset)

# Define an async function to test the LLM
#async def test_llm_with_prompts():
    #results = []
    #for prompt in prompts:
        # Call the custom LLM with each prompt
        #result = await custom_llm.generate_text(prompt)
        #results.append(result)
        # Debugging: Print each result
        #print(f"LLM Output for prompt: {result[:900]}")  # Truncate to avoid overly large output
    #return results

def is_json(myjson):
    try:
        json_object = json.loads(myjson)
    except ValueError as e:
        return False
    return True


#async def test_llm_with_prompts():
    # results = []
    # for prompt in prompts:
    #     # Call the custom LLM with each prompt
    #     llm_result = await custom_llm.generate_text(prompt=prompt)
        
    #     # Extract the text from LLMResult
    #     generated_text = llm_result.generations[0][0].text  # Access the first generation
        
    #     results.append(generated_text)
        
    #     # Debugging: Print each result (truncate to avoid overly large output)
    #     print(f"LLM Output for prompt: {generated_text[:900]}")  # Truncate for display
    # return results

if __name__ == "__main__":

    asyncio.run(run())
      # Ensure proper execution of async calls
    #all_results = asyncio.run(test_llm_with_prompts())
    #print(f"Total Results: {len(all_results)}")  # Confirm all prompts were processed

    embedding_dict=embedding_dataset

    ragas_embeddings = CustomHuggingFaceRagasEmbeddings(model_name='distilbert-base-uncased', custome_embedding= embedding_dict)
    #custom_llm =CustomRagasLLM(api_key=None)

    # Define the evaluation metrics
    metrics = [context_utilization]


    # Run the evaluation
    evaluation_report = evaluate(data_set, metrics=metrics, embeddings=ragas_embeddings, llm=llm ,is_async=True)  # Pass your custom LLM here

[jjmachan]

hey @amin-kh96 #1170 support is not there as of now sadly