diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs index 137c4bbf..9813b456 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs @@ -95,7 +95,12 @@ engine.CurrentBuild.JobState is BuildJobState.Pending await UpdateTrainJobStatus( platformService, engine.CurrentBuild.BuildId, - new ProgressStatus(step: 0, percentCompleted: 0.0), + new ProgressStatus( + step: 0, + percentCompleted: 0.0, + fineTuneProgress: 0.0, + inferenceProgress: 0.0 + ), //CurrentBuild.BuildId should always equal the corresponding task.Name queuePositionsPerEngineType[engine.Type][engine.CurrentBuild.BuildId] + 1, cancellationToken @@ -130,13 +135,26 @@ or ClearMLTaskStatus.Completed case ClearMLTaskStatus.InProgress: { double? percentCompleted = null; + double? fineTuneProgress = null; + double? inferenceProgress = null; + if (task.Runtime.TryGetValue("progress", out string? progressStr)) percentCompleted = int.Parse(progressStr, CultureInfo.InvariantCulture) / 100.0; + if (task.Runtime.TryGetValue("finetune", out string? fineTuneStr)) + fineTuneProgress = int.Parse(fineTuneStr, CultureInfo.InvariantCulture) / 100.0; + if (task.Runtime.TryGetValue("inference", out string? inferenceStr)) + inferenceProgress = int.Parse(inferenceStr, CultureInfo.InvariantCulture) / 100.0; task.Runtime.TryGetValue("message", out string? message); await UpdateTrainJobStatus( platformService, engine.CurrentBuild.BuildId, - new ProgressStatus(task.LastIteration ?? 0, percentCompleted, message), + new ProgressStatus( + task.LastIteration ?? 0, + percentCompleted, + fineTuneProgress, + inferenceProgress, + message + ), queueDepth: 0, cancellationToken ); @@ -149,7 +167,13 @@ await UpdateTrainJobStatus( await UpdateTrainJobStatus( platformService, engine.CurrentBuild.BuildId, - new ProgressStatus(task.LastIteration ?? 0, percentCompleted: 1.0, message), + new ProgressStatus( + task.LastIteration ?? 0, + percentCompleted: 1.0, + fineTuneProgress: 0.5, + inferenceProgress: 0.5, + message + ), queueDepth: 0, cancellationToken ); diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformService.cs index 6de64575..69576c36 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformService.cs @@ -85,6 +85,10 @@ public async Task UpdateBuildStatusAsync( var request = new UpdateBuildStatusRequest { BuildId = buildId, Step = progressStatus.Step }; if (progressStatus.PercentCompleted.HasValue) request.PercentCompleted = progressStatus.PercentCompleted.Value; + if (progressStatus.FineTuneProgress.HasValue) + request.FineTuneProgress = progressStatus.FineTuneProgress.Value; + if (progressStatus.InferenceProgress.HasValue) + request.InferenceProgress = progressStatus.InferenceProgress.Value; if (progressStatus.Message is not null) request.Message = progressStatus.Message; if (queueDepth is not null) diff --git a/src/Serval/src/Serval.Client/Client.g.cs b/src/Serval/src/Serval.Client/Client.g.cs index 1c372398..588eaab0 100644 --- a/src/Serval/src/Serval.Client/Client.g.cs +++ b/src/Serval/src/Serval.Client/Client.g.cs @@ -7763,6 +7763,12 @@ public partial class TranslationBuild [Newtonsoft.Json.JsonProperty("percentCompleted", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public double? PercentCompleted { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("fineTuneProgress", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public double? FineTuneProgress { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("inferenceProgress", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public double? InferenceProgress { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("message", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public string? Message { get; set; } = default!; diff --git a/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/platform.proto b/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/platform.proto index 235e884d..2ad035a1 100644 --- a/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/platform.proto +++ b/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/platform.proto @@ -21,8 +21,10 @@ message UpdateBuildStatusRequest { string build_id = 1; int32 step = 2; optional double percent_completed = 3; - optional string message = 4; - optional int32 queue_depth = 5; + optional double fine_tune_progress = 4; + optional double inference_progress = 5; + optional string message = 6; + optional int32 queue_depth = 7; } message BuildStartedRequest { diff --git a/src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs b/src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs index 0d21ab1a..f9327a22 100644 --- a/src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs +++ b/src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Contracts; +namespace Serval.Translation.Contracts; public record TranslationBuildDto { @@ -11,6 +11,9 @@ public record TranslationBuildDto public IReadOnlyList? Pretranslate { get; init; } public required int Step { get; init; } public double? PercentCompleted { get; init; } + public double? FineTuneProgress { get; init; } + public double? InferenceProgress { get; init; } + public string? Message { get; init; } public int? QueueDepth { get; init; } diff --git a/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs b/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs index 9f254a94..8756dd60 100644 --- a/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs +++ b/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs @@ -1596,6 +1596,8 @@ private TranslationBuildDto Map(Build source) Pretranslate = source.Pretranslate?.Select(s => Map(source.EngineRef, s)).ToList(), Step = source.Step, PercentCompleted = source.PercentCompleted, + FineTuneProgress = source.FineTuneProgress, + InferenceProgress = source.InferenceProgress, Message = source.Message, QueueDepth = source.QueueDepth, State = source.State, diff --git a/src/Serval/src/Serval.Translation/Models/Build.cs b/src/Serval/src/Serval.Translation/Models/Build.cs index 04f15fdd..1e21b851 100644 --- a/src/Serval/src/Serval.Translation/Models/Build.cs +++ b/src/Serval/src/Serval.Translation/Models/Build.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Models; +namespace Serval.Translation.Models; public record Build : IInitializableEntity { @@ -10,6 +10,8 @@ public record Build : IInitializableEntity public IReadOnlyList? Pretranslate { get; init; } public int Step { get; init; } public double? PercentCompleted { get; init; } + public double? FineTuneProgress { get; init; } + public double? InferenceProgress { get; init; } public string? Message { get; init; } public int? QueueDepth { get; init; } public JobState State { get; init; } = JobState.Pending; diff --git a/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs b/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs index 14f1444c..861bc990 100644 --- a/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs +++ b/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs @@ -218,6 +218,8 @@ await _dataAccessContext.WithTransactionAsync( u.Set(b => b.Message, "Restarting") .Set(b => b.Step, 0) .Set(b => b.PercentCompleted, 0) + .Set(b => b.FineTuneProgress, 0) + .Set(b => b.InferenceProgress, 0) .Set(b => b.State, JobState.Pending), cancellationToken: ct ); @@ -254,6 +256,20 @@ await _builds.UpdateAsync( Math.Round(request.PercentCompleted, 4, MidpointRounding.AwayFromZero) ); } + if (request.HasFineTuneProgress) + { + u.Set( + b => b.FineTuneProgress, + Math.Round(request.FineTuneProgress, 4, MidpointRounding.AwayFromZero) + ); + } + if (request.HasInferenceProgress) + { + u.Set( + b => b.InferenceProgress, + Math.Round(request.InferenceProgress, 4, MidpointRounding.AwayFromZero) + ); + } if (request.HasMessage) u.Set(b => b.Message, request.Message); if (request.HasQueueDepth) diff --git a/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs index 4ab47e10..852913f7 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs @@ -83,12 +83,16 @@ await env.PlatformService.UpdateBuildStatus( { BuildId = "b0", QueueDepth = 1, - PercentCompleted = 0.5 + PercentCompleted = 0.5, + FineTuneProgress = 0.25, + InferenceProgress = 0.3, }, env.ServerCallContext ); Assert.That(env.Builds.Get("b0").QueueDepth, Is.EqualTo(1)); Assert.That(env.Builds.Get("b0").PercentCompleted, Is.EqualTo(0.5)); + Assert.That(env.Builds.Get("b0").FineTuneProgress, Is.EqualTo(0.25)); + Assert.That(env.Builds.Get("b0").InferenceProgress, Is.EqualTo(0.3)); } [Test]