Skip to content

Commit

Permalink
in the road
Browse files Browse the repository at this point in the history
  • Loading branch information
@sdcb
sdcb committed Feb 7, 2025
1 parent c92e959 commit 8f2f335
Show file tree
Hide file tree
Showing 14 changed files with 58 additions and 26 deletions.
6 changes: 3 additions & 3 deletions src/BE/Controllers/Chats/Chats/ChatController.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -402,8 +402,8 @@ private static async Task<ChatSpanResponse> ProcessChatSpan(
using ChatService s = chatFactory.CreateChatService(userModel.Model);
await foreach (InternalChatSegment seg in icc.Run(userBalance.Balance, userModel, s.ChatStreamedFEProcessed(messageToSend, cco, extraDetails, cancellationToken)))
{
if (seg.TextSegment == string.Empty) continue;
await writer.WriteAsync(SseResponseLine.Segment(span.Id, seg.TextSegment), cancellationToken);
if (seg.Segment == string.Empty) continue;
await writer.WriteAsync(SseResponseLine.Segment(span.Id, seg.Segment), cancellationToken);

if (cancellationToken.IsCancellationRequested)
{
Expand Down Expand Up @@ -448,7 +448,7 @@ private static async Task<ChatSpanResponse> ProcessChatSpan(
ChatRoleId = (byte)DBChatRole.Assistant,
MessageContents =
[
MessageContent.FromText(icc.FullResponse.TextSegment),
MessageContent.FromText(icc.FullResponse.Segment),
],
SpanId = span.Id,
CreatedAt = DateTime.UtcNow,
Expand Down
5 changes: 4 additions & 1 deletion src/BE/Controllers/OpenAICompatible/Dtos/ChatCompletionChunk.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,10 @@ namespace Chats.BE.Controllers.OpenAICompatible.Dtos;
public record Delta
{
[JsonPropertyName("content")]
public required string Content { get; init; }
public required string? Content { get; init; }

[JsonPropertyName("reasoning_content"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public required string? ReasoningContent { get; init; }
}

public record DeltaChoice
Expand Down
5 changes: 4 additions & 1 deletion src/BE/Controllers/OpenAICompatible/Dtos/FullChatCompletion.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,10 @@ public record ResponseMessage
public required string Role { get; init; }

[JsonPropertyName("content")]
public required string Content { get; init; }
public required string? Content { get; init; }

[JsonPropertyName("reasoning_content"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public required string? ReasoningContent { get; init; }

[JsonPropertyName("refusal")]
public object? Refusal { get; init; }
Expand Down
2 changes: 1 addition & 1 deletion src/BE/Controllers/OpenAICompatible/OpenAICompatibleController.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ public async Task<ActionResult> ChatCompletion([FromBody] JsonObject json, [From
{
await foreach (InternalChatSegment seg in icc.Run(userBalance.Balance, userModel, s.ChatStreamedSimulated(cco.Stream, [.. cco.Messages], cco.ToCleanCco(), cancellationToken)))
{
if (seg.TextSegment == string.Empty) continue;
if (seg.Segment == string.Empty) continue;

if (cco.Stream)
{
Expand Down
4 changes: 2 additions & 2 deletions src/BE/Services/Models/ChatService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,14 +38,14 @@ public virtual async Task<ChatSegment> Chat(IReadOnlyList<ChatMessage> messages,
await foreach (ChatSegment seg in ChatStreamed(messages, options, cancellationToken))
{
lastSegment = seg;
result.Append(seg.TextSegment);
result.Append(seg.Segment);
}

return new ChatSegment()
{
Usage = lastSegment?.Usage,
FinishReason = lastSegment?.FinishReason,
TextSegment = result.ToString(),
Segment = result.ToString(),
};
}

Expand Down
4 changes: 2 additions & 2 deletions src/BE/Services/Models/ChatServiceExtensions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ public async IAsyncEnumerable<InternalChatSegment> ChatStreamedSimulated(bool su
yield return seg.ToInternal(() => new Dtos.ChatTokenUsage
{
InputTokens = inputTokens,
OutputTokens = outputTokens += Tokenizer.CountTokens(seg.TextSegment),
OutputTokens = outputTokens += Tokenizer.CountTokens(seg.Segment),
ReasoningTokens = 0
});
}
Expand All @@ -42,7 +42,7 @@ public async IAsyncEnumerable<InternalChatSegment> ChatStreamedSimulated(bool su
yield return seg.ToInternal(() => new Dtos.ChatTokenUsage()
{
InputTokens = inputTokens,
OutputTokens = outputTokens += Tokenizer.CountTokens(seg.TextSegment),
OutputTokens = outputTokens += Tokenizer.CountTokens(seg.Segment),
ReasoningTokens = 0
});
}
Expand Down
4 changes: 2 additions & 2 deletions src/BE/Services/Models/ChatServices/DashScope/DashScopeChatService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public override async IAsyncEnumerable<ChatSegment> ChatStreamed(IReadOnlyList<O
{
yield return new ChatSegment
{
TextSegment = resp.Output,
Segment = resp.Output,
FinishReason = null,
Usage = resp.Usage != null ? new Dtos.ChatTokenUsage
{
Expand All @@ -64,7 +64,7 @@ public override async IAsyncEnumerable<ChatSegment> ChatStreamed(IReadOnlyList<O
{
yield return new ChatSegment
{
TextSegment = resp.Output.Choices[0].Message.Content,
Segment = resp.Output.Choices[0].Message.Content,
FinishReason = ToFinishReason(resp.Output.Choices[0].FinishReason),
Usage = resp.Usage != null ? new Dtos.ChatTokenUsage
{
Expand Down
2 changes: 1 addition & 1 deletion src/BE/Services/Models/ChatServices/Hunyuan/HunyuanChatService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ public override async IAsyncEnumerable<ChatSegment> ChatStreamed(IReadOnlyList<C

yield return new ChatSegment
{
TextSegment = seg.Choices[0].Delta.Content,
Segment = seg.Choices[0].Delta.Content,
FinishReason = ToFinishReason(seg.Choices[0].FinishReason),
Usage = new Dtos.ChatTokenUsage
{
Expand Down
22 changes: 20 additions & 2 deletions src/BE/Services/Models/ChatServices/OpenAI/OpenAIChatService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using Chats.BE.DB;
using System.ClientModel.Primitives;
using System.Text.Json;
using Chats.BE.Services.UrlEncryption;

namespace Chats.BE.Services.Models.ChatServices.OpenAI;

Expand All @@ -29,15 +30,32 @@ public OpenAIChatService(Model model, ChatClient chatClient) : base(model)
_chatClient = chatClient;
}

static IEnumerable ChoicesAccessor()
{
//IEnumerable choices = delta.Uncapsulate().Choices;
//Dictionary<string, BinaryData>? sad = choices.Cast<object>().FirstOrDefault()?.Uncapsulate().Delta.SerializedAdditionalRawData;
//if (sad != null && sad.Any() && sad.TryGetValue("reasoning_content", out BinaryData? rc))
//{
// string? think = rc.ToObjectFromJson<string>();
// if (think != null)
// {
// Console.Write(Util.WithStyle(rc.ToObjectFromJson<string>(), "color: green"));
// }
//}
}

public override async IAsyncEnumerable<ChatSegment> ChatStreamed(IReadOnlyList<ChatMessage> messages, ChatCompletionOptions options, [EnumeratorCancellation] CancellationToken cancellationToken)
{
await foreach (StreamingChatCompletionUpdate delta in _chatClient.CompleteChatStreamingAsync(messages, options, cancellationToken))
{
if (delta.ContentUpdate.Count == 0 && delta.Usage == null) continue;

string? segment = delta.ContentUpdate.FirstOrDefault()?.Text;
string? reasoningSegment =

yield return new ChatSegment
{
TextSegment = delta.ContentUpdate.FirstOrDefault()?.Text ?? "",
Segment = segment,
FinishReason = delta.FinishReason,
Usage = delta.Usage != null ? new Dtos.ChatTokenUsage()
{
Expand Down Expand Up @@ -65,7 +83,7 @@ public override async Task<ChatSegment> Chat(IReadOnlyList<ChatMessage> messages
ChatCompletion delta = cc.Value;
return new ChatSegment
{
TextSegment = delta.Content[0].Text,
Segment = delta.Content[0].Text,
FinishReason = delta.FinishReason,
Usage = delta.Usage != null ? GetUsage(delta.Usage) : null,
};
Expand Down
2 changes: 1 addition & 1 deletion src/BE/Services/Models/ChatServices/QianFan/QianFanChatService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ public override async IAsyncEnumerable<ChatSegment> ChatStreamed(IReadOnlyList<O
{
yield return new ChatSegment
{
TextSegment = chatResponse.Result,
Segment = chatResponse.Result,
FinishReason = ToFinishReason(chatResponse.FinishReason),
Usage = new Dtos.ChatTokenUsage
{
Expand Down
2 changes: 1 addition & 1 deletion src/BE/Services/Models/ChatServices/Test/TestChatService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ public override async IAsyncEnumerable<ChatSegment> ChatStreamed(
int outputTokens = Tokenizer.CountTokens(outputed.ToString());
yield return new ChatSegment()
{
TextSegment = combined.ToString(),
Segment = combined.ToString(),
Usage = new Dtos.ChatTokenUsage()
{
InputTokens = inputTokens,
Expand Down
10 changes: 7 additions & 3 deletions src/BE/Services/Models/Dtos/ChatSegment.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@ public record ChatSegment
{
public required ChatFinishReason? FinishReason { get; init; }

public required string TextSegment { get; init; }
public required string? Segment { get; init; }

public required string? ReasoningSegment { get; init; }

public required ChatTokenUsage? Usage { get; init; }

Expand All @@ -18,7 +20,8 @@ public InternalChatSegment ToInternal(Func<ChatTokenUsage> usageCalculator)
{
Usage = Usage,
FinishReason = FinishReason,
TextSegment = TextSegment,
Segment = Segment,
ReasoningSegment = ReasoningSegment,
IsUsageReliable = true,
IsFromUpstream = true,
};
Expand All @@ -29,7 +32,8 @@ public InternalChatSegment ToInternal(Func<ChatTokenUsage> usageCalculator)
{
Usage = Usage ?? usageCalculator(),
FinishReason = FinishReason,
TextSegment = TextSegment,
Segment = Segment,
ReasoningSegment = ReasoningSegment,
IsUsageReliable = false,
IsFromUpstream = true,
};
Expand Down
12 changes: 8 additions & 4 deletions src/BE/Services/Models/Dtos/InternalChatSegment.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@ public record InternalChatSegment
{
public required ChatFinishReason? FinishReason { get; init; }

public required string TextSegment { get; init; }
public required string? Segment { get; init; }

public required string? ReasoningSegment { get; init; }

public required ChatTokenUsage Usage { get; init; }

Expand All @@ -19,7 +21,8 @@ public record InternalChatSegment
{
Usage = ChatTokenUsage.Zero,
FinishReason = null,
TextSegment = string.Empty,
Segment = null,
ReasoningSegment = null,
IsUsageReliable = false,
IsFromUpstream = false,
};
Expand Down Expand Up @@ -84,7 +87,7 @@ internal ChatCompletionChunk ToOpenAIChunk(string modelName, string traceId)
[
new DeltaChoice
{
Delta = new Delta { Content = TextSegment },
Delta = new Delta { Content = Segment, ReasoningContent = ReasoningSegment },
FinishReason = GetFinishReasonText(),
Index = 0,
Logprobs = null,
Expand All @@ -110,7 +113,8 @@ internal FullChatCompletion ToOpenAIFullChat(string modelName, string traceId)
Message = new ResponseMessage
{
Role = "system",
Content = TextSegment,
Content = Segment,
ReasoningContent = ReasoningSegment,
Refusal = null,
}
}
Expand Down
4 changes: 2 additions & 2 deletions src/BE/Services/Models/InChatContext.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ public async IAsyncEnumerable<InternalChatSegment> Run(decimal userBalance, User
if (_firstResponseTick == 0) _firstResponseTick = Stopwatch.GetTimestamp();
}
_lastSegment = seg;
_fullResult.Append(seg.TextSegment);
_fullResult.Append(seg.Segment);

UserModelBalanceCost currentCost = calculator.GetNewBalance(seg.Usage.InputTokens, seg.Usage.OutputTokens, priceConfig);
if (!currentCost.IsSufficient)
Expand All @@ -71,7 +71,7 @@ public async IAsyncEnumerable<InternalChatSegment> Run(decimal userBalance, User
}
}

public InternalChatSegment FullResponse => _lastSegment with { TextSegment = _fullResult.ToString() };
public InternalChatSegment FullResponse => _lastSegment with { Segment = _fullResult.ToString() };

public UserModelUsage ToUserModelUsage(int userId, ClientInfo clientInfo, bool isApi)
{
Expand Down

0 comments on commit 8f2f335

Please sign in to comment.