[Feature] Add support for custom streaming callback

embedchain/apps/app.py

@@ -10,7 +10,7 @@
 from embedchain.embedder.base import BaseEmbedder
 from embedchain.embedder.openai import OpenAIEmbedder
 from embedchain.factory import EmbedderFactory, LlmFactory, VectorDBFactory
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.llm.base import BaseLlm
 from embedchain.llm.openai import OpenAILlm
 from embedchain.utils import validate_yaml_config

embedchain/bots/base.py

@@ -3,8 +3,8 @@
 from embedchain import Pipeline as App
 from embedchain.config import AddConfig, BaseLlmConfig, PipelineConfig
 from embedchain.embedder.openai import OpenAIEmbedder
-from embedchain.helper.json_serializable import (JSONSerializable,
-                                                 register_deserializable)
+from embedchain.helpers.json_serializable import (JSONSerializable,
+                                                  register_deserializable)
 from embedchain.llm.openai import OpenAILlm
 from embedchain.vectordb.chroma import ChromaDB
 

embedchain/bots/discord.py

@@ -2,7 +2,7 @@
 import logging
 import os
 
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 from .base import BaseBot
 

embedchain/bots/poe.py

@@ -3,7 +3,7 @@
 import os
 from typing import List, Optional
 
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 from .base import BaseBot
 

embedchain/bots/slack.py

@@ -5,7 +5,7 @@
 import sys
 
 from embedchain import App
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 from .base import BaseBot
 

embedchain/bots/whatsapp.py

@@ -4,7 +4,7 @@
 import signal
 import sys
 
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 from .base import BaseBot
 

embedchain/chunkers/base_chunker.py

@@ -1,6 +1,6 @@
 import hashlib
 
-from embedchain.helper.json_serializable import JSONSerializable
+from embedchain.helpers.json_serializable import JSONSerializable
 from embedchain.models.data_type import DataType
 
 

embedchain/chunkers/common_chunker.py

@@ -4,7 +4,7 @@
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/discourse.py

@@ -4,7 +4,7 @@
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/docs_site.py

@@ -4,7 +4,7 @@
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/docx_file.py

@@ -4,7 +4,7 @@
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/gmail.py

@@ -4,7 +4,7 @@
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/json.py

@@ -4,7 +4,7 @@
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/mdx.py

@@ -4,7 +4,7 @@
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/mysql.py

@@ -4,7 +4,7 @@
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/notion.py

@@ -4,7 +4,7 @@
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/pdf_file.py

@@ -4,7 +4,7 @@
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/postgres.py

@@ -4,7 +4,7 @@
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/qna_pair.py

@@ -4,7 +4,7 @@
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/sitemap.py

@@ -4,7 +4,7 @@
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/slack.py

@@ -4,7 +4,7 @@
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/substack.py

@@ -4,7 +4,7 @@
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/text.py

@@ -4,7 +4,7 @@
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/unstructured_file.py

@@ -4,7 +4,7 @@
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/web_page.py

@@ -4,7 +4,7 @@
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/xml.py

@@ -4,7 +4,7 @@
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/youtube_video.py

@@ -4,7 +4,7 @@
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/config/add_config.py

@@ -3,7 +3,7 @@
 from typing import Callable, Optional
 
 from embedchain.config.base_config import BaseConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/config/apps/app_config.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 from .base_app_config import BaseAppConfig
 

embedchain/config/apps/base_app_config.py

@@ -2,7 +2,7 @@
 from typing import Optional
 
 from embedchain.config.base_config import BaseConfig
-from embedchain.helper.json_serializable import JSONSerializable
+from embedchain.helpers.json_serializable import JSONSerializable
 from embedchain.vectordb.base import BaseVectorDB
 
 

embedchain/config/base_config.py

@@ -1,6 +1,6 @@
 from typing import Any, Dict
 
-from embedchain.helper.json_serializable import JSONSerializable
+from embedchain.helpers.json_serializable import JSONSerializable
 
 
 class BaseConfig(JSONSerializable):

embedchain/config/embedder/base.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/config/llm/base.py

@@ -1,9 +1,9 @@
 import re
 from string import Template
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional
 
 from embedchain.config.base_config import BaseConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 DEFAULT_PROMPT = """
   Use the following pieces of context to answer the query at the end.
@@ -68,6 +68,7 @@ def __init__(
         system_prompt: Optional[str] = None,
         where: Dict[str, Any] = None,
         query_type: Optional[str] = None,
+        callbacks: Optional[List] = None,
     ):
         """
         Initializes a configuration class instance for the LLM.
@@ -98,6 +99,8 @@ def __init__(
         :type system_prompt: Optional[str], optional
         :param where: A dictionary of key-value pairs to filter the database results., defaults to None
         :type where: Dict[str, Any], optional
+        :param callbacks: Langchain callback functions to use, defaults to None
+        :type callbacks: Optional[List], optional
         :raises ValueError: If the template is not valid as template should
         contain $context and $query (and optionally $history)
         :raises ValueError: Stream is not boolean
@@ -113,6 +116,7 @@ def __init__(
         self.deployment_name = deployment_name
         self.system_prompt = system_prompt
         self.query_type = query_type
+        self.callbacks = callbacks
 
         if type(template) is str:
             template = Template(template)

embedchain/config/pipeline_config.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 from .apps.base_app_config import BaseAppConfig
 

embedchain/config/vectordb/chroma.py

@@ -1,7 +1,7 @@
 from typing import Optional
 
 from embedchain.config.vectordb.base import BaseVectorDbConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/config/vectordb/elasticsearch.py

@@ -2,7 +2,7 @@
 from typing import Dict, List, Optional, Union
 
 from embedchain.config.vectordb.base import BaseVectorDbConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/config/vectordb/opensearch.py

@@ -1,7 +1,7 @@
 from typing import Dict, Optional, Tuple
 
 from embedchain.config.vectordb.base import BaseVectorDbConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/config/vectordb/pinecone.py

@@ -1,7 +1,7 @@
 from typing import Dict, Optional
 
 from embedchain.config.vectordb.base import BaseVectorDbConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/config/vectordb/qdrant.py

@@ -1,7 +1,7 @@
 from typing import Dict, Optional
 
 from embedchain.config.vectordb.base import BaseVectorDbConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/config/vectordb/weaviate.py

@@ -1,7 +1,7 @@
 from typing import Dict, Optional
 
 from embedchain.config.vectordb.base import BaseVectorDbConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/config/vectordb/zilliz.py

@@ -2,7 +2,7 @@
 from typing import Optional
 
 from embedchain.config.vectordb.base import BaseVectorDbConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable