From 8f2f33534daa1d1e5d9fc6143b4abe44f0d2e1d2 Mon Sep 17 00:00:00 2001 From: sdcb Date: Fri, 7 Feb 2025 19:26:39 +0800 Subject: [PATCH] in the road --- .../Controllers/Chats/Chats/ChatController.cs | 6 ++--- .../Dtos/ChatCompletionChunk.cs | 5 ++++- .../Dtos/FullChatCompletion.cs | 5 ++++- .../OpenAICompatibleController.cs | 2 +- src/BE/Services/Models/ChatService.cs | 4 ++-- .../Services/Models/ChatServiceExtensions.cs | 4 ++-- .../DashScope/DashScopeChatService.cs | 4 ++-- .../Hunyuan/HunyuanChatService.cs | 2 +- .../ChatServices/OpenAI/OpenAIChatService.cs | 22 +++++++++++++++++-- .../QianFan/QianFanChatService.cs | 2 +- .../ChatServices/Test/TestChatService.cs | 2 +- src/BE/Services/Models/Dtos/ChatSegment.cs | 10 ++++++--- .../Models/Dtos/InternalChatSegment.cs | 12 ++++++---- src/BE/Services/Models/InChatContext.cs | 4 ++-- 14 files changed, 58 insertions(+), 26 deletions(-) diff --git a/src/BE/Controllers/Chats/Chats/ChatController.cs b/src/BE/Controllers/Chats/Chats/ChatController.cs index 279a318d..f092844b 100644 --- a/src/BE/Controllers/Chats/Chats/ChatController.cs +++ b/src/BE/Controllers/Chats/Chats/ChatController.cs @@ -402,8 +402,8 @@ private static async Task ProcessChatSpan( using ChatService s = chatFactory.CreateChatService(userModel.Model); await foreach (InternalChatSegment seg in icc.Run(userBalance.Balance, userModel, s.ChatStreamedFEProcessed(messageToSend, cco, extraDetails, cancellationToken))) { - if (seg.TextSegment == string.Empty) continue; - await writer.WriteAsync(SseResponseLine.Segment(span.Id, seg.TextSegment), cancellationToken); + if (seg.Segment == string.Empty) continue; + await writer.WriteAsync(SseResponseLine.Segment(span.Id, seg.Segment), cancellationToken); if (cancellationToken.IsCancellationRequested) { @@ -448,7 +448,7 @@ private static async Task ProcessChatSpan( ChatRoleId = (byte)DBChatRole.Assistant, MessageContents = [ - MessageContent.FromText(icc.FullResponse.TextSegment), + MessageContent.FromText(icc.FullResponse.Segment), ], SpanId = span.Id, CreatedAt = DateTime.UtcNow, diff --git a/src/BE/Controllers/OpenAICompatible/Dtos/ChatCompletionChunk.cs b/src/BE/Controllers/OpenAICompatible/Dtos/ChatCompletionChunk.cs index 1546b628..b5c07877 100644 --- a/src/BE/Controllers/OpenAICompatible/Dtos/ChatCompletionChunk.cs +++ b/src/BE/Controllers/OpenAICompatible/Dtos/ChatCompletionChunk.cs @@ -5,7 +5,10 @@ namespace Chats.BE.Controllers.OpenAICompatible.Dtos; public record Delta { [JsonPropertyName("content")] - public required string Content { get; init; } + public required string? Content { get; init; } + + [JsonPropertyName("reasoning_content"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public required string? ReasoningContent { get; init; } } public record DeltaChoice diff --git a/src/BE/Controllers/OpenAICompatible/Dtos/FullChatCompletion.cs b/src/BE/Controllers/OpenAICompatible/Dtos/FullChatCompletion.cs index a97d3812..37197751 100644 --- a/src/BE/Controllers/OpenAICompatible/Dtos/FullChatCompletion.cs +++ b/src/BE/Controllers/OpenAICompatible/Dtos/FullChatCompletion.cs @@ -47,7 +47,10 @@ public record ResponseMessage public required string Role { get; init; } [JsonPropertyName("content")] - public required string Content { get; init; } + public required string? Content { get; init; } + + [JsonPropertyName("reasoning_content"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public required string? ReasoningContent { get; init; } [JsonPropertyName("refusal")] public object? Refusal { get; init; } diff --git a/src/BE/Controllers/OpenAICompatible/OpenAICompatibleController.cs b/src/BE/Controllers/OpenAICompatible/OpenAICompatibleController.cs index 4e46e035..2e9b14a5 100644 --- a/src/BE/Controllers/OpenAICompatible/OpenAICompatibleController.cs +++ b/src/BE/Controllers/OpenAICompatible/OpenAICompatibleController.cs @@ -48,7 +48,7 @@ public async Task ChatCompletion([FromBody] JsonObject json, [From { await foreach (InternalChatSegment seg in icc.Run(userBalance.Balance, userModel, s.ChatStreamedSimulated(cco.Stream, [.. cco.Messages], cco.ToCleanCco(), cancellationToken))) { - if (seg.TextSegment == string.Empty) continue; + if (seg.Segment == string.Empty) continue; if (cco.Stream) { diff --git a/src/BE/Services/Models/ChatService.cs b/src/BE/Services/Models/ChatService.cs index be87d92d..0b147aa4 100644 --- a/src/BE/Services/Models/ChatService.cs +++ b/src/BE/Services/Models/ChatService.cs @@ -38,14 +38,14 @@ public virtual async Task Chat(IReadOnlyList messages, await foreach (ChatSegment seg in ChatStreamed(messages, options, cancellationToken)) { lastSegment = seg; - result.Append(seg.TextSegment); + result.Append(seg.Segment); } return new ChatSegment() { Usage = lastSegment?.Usage, FinishReason = lastSegment?.FinishReason, - TextSegment = result.ToString(), + Segment = result.ToString(), }; } diff --git a/src/BE/Services/Models/ChatServiceExtensions.cs b/src/BE/Services/Models/ChatServiceExtensions.cs index 95228582..4935aa0a 100644 --- a/src/BE/Services/Models/ChatServiceExtensions.cs +++ b/src/BE/Services/Models/ChatServiceExtensions.cs @@ -31,7 +31,7 @@ public async IAsyncEnumerable ChatStreamedSimulated(bool su yield return seg.ToInternal(() => new Dtos.ChatTokenUsage { InputTokens = inputTokens, - OutputTokens = outputTokens += Tokenizer.CountTokens(seg.TextSegment), + OutputTokens = outputTokens += Tokenizer.CountTokens(seg.Segment), ReasoningTokens = 0 }); } @@ -42,7 +42,7 @@ public async IAsyncEnumerable ChatStreamedSimulated(bool su yield return seg.ToInternal(() => new Dtos.ChatTokenUsage() { InputTokens = inputTokens, - OutputTokens = outputTokens += Tokenizer.CountTokens(seg.TextSegment), + OutputTokens = outputTokens += Tokenizer.CountTokens(seg.Segment), ReasoningTokens = 0 }); } diff --git a/src/BE/Services/Models/ChatServices/DashScope/DashScopeChatService.cs b/src/BE/Services/Models/ChatServices/DashScope/DashScopeChatService.cs index 17c4bc51..7ba88cec 100644 --- a/src/BE/Services/Models/ChatServices/DashScope/DashScopeChatService.cs +++ b/src/BE/Services/Models/ChatServices/DashScope/DashScopeChatService.cs @@ -46,7 +46,7 @@ public override async IAsyncEnumerable ChatStreamed(IReadOnlyList ChatStreamed(IReadOnlyList ChatStreamed(IReadOnlyList? sad = choices.Cast().FirstOrDefault()?.Uncapsulate().Delta.SerializedAdditionalRawData; + //if (sad != null && sad.Any() && sad.TryGetValue("reasoning_content", out BinaryData? rc)) + //{ + // string? think = rc.ToObjectFromJson(); + // if (think != null) + // { + // Console.Write(Util.WithStyle(rc.ToObjectFromJson(), "color: green")); + // } + //} + } + public override async IAsyncEnumerable ChatStreamed(IReadOnlyList messages, ChatCompletionOptions options, [EnumeratorCancellation] CancellationToken cancellationToken) { await foreach (StreamingChatCompletionUpdate delta in _chatClient.CompleteChatStreamingAsync(messages, options, cancellationToken)) { if (delta.ContentUpdate.Count == 0 && delta.Usage == null) continue; + string? segment = delta.ContentUpdate.FirstOrDefault()?.Text; + string? reasoningSegment = + yield return new ChatSegment { - TextSegment = delta.ContentUpdate.FirstOrDefault()?.Text ?? "", + Segment = segment, FinishReason = delta.FinishReason, Usage = delta.Usage != null ? new Dtos.ChatTokenUsage() { @@ -65,7 +83,7 @@ public override async Task Chat(IReadOnlyList messages ChatCompletion delta = cc.Value; return new ChatSegment { - TextSegment = delta.Content[0].Text, + Segment = delta.Content[0].Text, FinishReason = delta.FinishReason, Usage = delta.Usage != null ? GetUsage(delta.Usage) : null, }; diff --git a/src/BE/Services/Models/ChatServices/QianFan/QianFanChatService.cs b/src/BE/Services/Models/ChatServices/QianFan/QianFanChatService.cs index 378a96c8..63c11987 100644 --- a/src/BE/Services/Models/ChatServices/QianFan/QianFanChatService.cs +++ b/src/BE/Services/Models/ChatServices/QianFan/QianFanChatService.cs @@ -45,7 +45,7 @@ public override async IAsyncEnumerable ChatStreamed(IReadOnlyList ChatStreamed( int outputTokens = Tokenizer.CountTokens(outputed.ToString()); yield return new ChatSegment() { - TextSegment = combined.ToString(), + Segment = combined.ToString(), Usage = new Dtos.ChatTokenUsage() { InputTokens = inputTokens, diff --git a/src/BE/Services/Models/Dtos/ChatSegment.cs b/src/BE/Services/Models/Dtos/ChatSegment.cs index 65607f8f..442c02eb 100644 --- a/src/BE/Services/Models/Dtos/ChatSegment.cs +++ b/src/BE/Services/Models/Dtos/ChatSegment.cs @@ -6,7 +6,9 @@ public record ChatSegment { public required ChatFinishReason? FinishReason { get; init; } - public required string TextSegment { get; init; } + public required string? Segment { get; init; } + + public required string? ReasoningSegment { get; init; } public required ChatTokenUsage? Usage { get; init; } @@ -18,7 +20,8 @@ public InternalChatSegment ToInternal(Func usageCalculator) { Usage = Usage, FinishReason = FinishReason, - TextSegment = TextSegment, + Segment = Segment, + ReasoningSegment = ReasoningSegment, IsUsageReliable = true, IsFromUpstream = true, }; @@ -29,7 +32,8 @@ public InternalChatSegment ToInternal(Func usageCalculator) { Usage = Usage ?? usageCalculator(), FinishReason = FinishReason, - TextSegment = TextSegment, + Segment = Segment, + ReasoningSegment = ReasoningSegment, IsUsageReliable = false, IsFromUpstream = true, }; diff --git a/src/BE/Services/Models/Dtos/InternalChatSegment.cs b/src/BE/Services/Models/Dtos/InternalChatSegment.cs index 9493973f..9d595f29 100644 --- a/src/BE/Services/Models/Dtos/InternalChatSegment.cs +++ b/src/BE/Services/Models/Dtos/InternalChatSegment.cs @@ -7,7 +7,9 @@ public record InternalChatSegment { public required ChatFinishReason? FinishReason { get; init; } - public required string TextSegment { get; init; } + public required string? Segment { get; init; } + + public required string? ReasoningSegment { get; init; } public required ChatTokenUsage Usage { get; init; } @@ -19,7 +21,8 @@ public record InternalChatSegment { Usage = ChatTokenUsage.Zero, FinishReason = null, - TextSegment = string.Empty, + Segment = null, + ReasoningSegment = null, IsUsageReliable = false, IsFromUpstream = false, }; @@ -84,7 +87,7 @@ internal ChatCompletionChunk ToOpenAIChunk(string modelName, string traceId) [ new DeltaChoice { - Delta = new Delta { Content = TextSegment }, + Delta = new Delta { Content = Segment, ReasoningContent = ReasoningSegment }, FinishReason = GetFinishReasonText(), Index = 0, Logprobs = null, @@ -110,7 +113,8 @@ internal FullChatCompletion ToOpenAIFullChat(string modelName, string traceId) Message = new ResponseMessage { Role = "system", - Content = TextSegment, + Content = Segment, + ReasoningContent = ReasoningSegment, Refusal = null, } } diff --git a/src/BE/Services/Models/InChatContext.cs b/src/BE/Services/Models/InChatContext.cs index a56442d9..bf30b9d5 100644 --- a/src/BE/Services/Models/InChatContext.cs +++ b/src/BE/Services/Models/InChatContext.cs @@ -51,7 +51,7 @@ public async IAsyncEnumerable Run(decimal userBalance, User if (_firstResponseTick == 0) _firstResponseTick = Stopwatch.GetTimestamp(); } _lastSegment = seg; - _fullResult.Append(seg.TextSegment); + _fullResult.Append(seg.Segment); UserModelBalanceCost currentCost = calculator.GetNewBalance(seg.Usage.InputTokens, seg.Usage.OutputTokens, priceConfig); if (!currentCost.IsSufficient) @@ -71,7 +71,7 @@ public async IAsyncEnumerable Run(decimal userBalance, User } } - public InternalChatSegment FullResponse => _lastSegment with { TextSegment = _fullResult.ToString() }; + public InternalChatSegment FullResponse => _lastSegment with { Segment = _fullResult.ToString() }; public UserModelUsage ToUserModelUsage(int userId, ClientInfo clientInfo, bool isApi) {