diff --git a/comps/llms/src/doc-summarization/integrations/common.py b/comps/llms/src/doc-summarization/integrations/common.py index 934c80072..b989cf4e9 100644 --- a/comps/llms/src/doc-summarization/integrations/common.py +++ b/comps/llms/src/doc-summarization/integrations/common.py @@ -191,8 +191,11 @@ async def generate(self, input: DocSumChatCompletionRequest, client): if input.stream: import json + from langserve.serialization import WellKnownLCSerializer + _serializer = WellKnownLCSerializer() + def extract_llm_tokens(stream_output): op_data = _serializer.dumps({"ops": stream_output.ops}).decode("utf-8") logger.info(op_data) @@ -211,7 +214,7 @@ async def stream_generator(): if logflag: logger.info(data) - if data != '': + if data != "": yield f"data: {data}\n\n" yield "data: [DONE]\n\n"