Added to build properties

sillsdev · Feb 20, 2025 · 2ead129 · 2ead129
1 parent ffd8a1e
commit 2ead129
Show file tree

Hide file tree

Showing 9 changed files with 71 additions and 8 deletions.
diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs
@@ -95,7 +95,12 @@ engine.CurrentBuild.JobState is BuildJobState.Pending
                     await UpdateTrainJobStatus(
                         platformService,
                         engine.CurrentBuild.BuildId,
-                        new ProgressStatus(step: 0, percentCompleted: 0.0),
+                        new ProgressStatus(
+                            step: 0,
+                            percentCompleted: 0.0,
+                            fineTuneProgress: 0.0,
+                            inferenceProgress: 0.0
+                        ),
                         //CurrentBuild.BuildId should always equal the corresponding task.Name
                         queuePositionsPerEngineType[engine.Type][engine.CurrentBuild.BuildId] + 1,
                         cancellationToken
@@ -130,13 +135,26 @@ or ClearMLTaskStatus.Completed
                         case ClearMLTaskStatus.InProgress:
                         {
                             double? percentCompleted = null;
+                            double? fineTuneProgress = null;
+                            double? inferenceProgress = null;
+
                             if (task.Runtime.TryGetValue("progress", out string? progressStr))
                                 percentCompleted = int.Parse(progressStr, CultureInfo.InvariantCulture) / 100.0;
+                            if (task.Runtime.TryGetValue("finetune", out string? fineTuneStr))
+                                fineTuneProgress = int.Parse(fineTuneStr, CultureInfo.InvariantCulture) / 100.0;
+                            if (task.Runtime.TryGetValue("inference", out string? inferenceStr))
+                                inferenceProgress = int.Parse(inferenceStr, CultureInfo.InvariantCulture) / 100.0;
                             task.Runtime.TryGetValue("message", out string? message);
                             await UpdateTrainJobStatus(
                                 platformService,
                                 engine.CurrentBuild.BuildId,
-                                new ProgressStatus(task.LastIteration ?? 0, percentCompleted, message),
+                                new ProgressStatus(
+                                    task.LastIteration ?? 0,
+                                    percentCompleted,
+                                    fineTuneProgress,
+                                    inferenceProgress,
+                                    message
+                                ),
                                 queueDepth: 0,
                                 cancellationToken
                             );
@@ -149,7 +167,13 @@ await UpdateTrainJobStatus(
                             await UpdateTrainJobStatus(
                                 platformService,
                                 engine.CurrentBuild.BuildId,
-                                new ProgressStatus(task.LastIteration ?? 0, percentCompleted: 1.0, message),
+                                new ProgressStatus(
+                                    task.LastIteration ?? 0,
+                                    percentCompleted: 1.0,
+                                    fineTuneProgress: 0.5,
+                                    inferenceProgress: 0.5,
+                                    message
+                                ),
                                 queueDepth: 0,
                                 cancellationToken
                             );

diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformService.cs
@@ -85,6 +85,10 @@ public async Task UpdateBuildStatusAsync(
         var request = new UpdateBuildStatusRequest { BuildId = buildId, Step = progressStatus.Step };
         if (progressStatus.PercentCompleted.HasValue)
             request.PercentCompleted = progressStatus.PercentCompleted.Value;
+        if (progressStatus.FineTuneProgress.HasValue)
+            request.FineTuneProgress = progressStatus.FineTuneProgress.Value;
+        if (progressStatus.InferenceProgress.HasValue)
+            request.InferenceProgress = progressStatus.InferenceProgress.Value;
         if (progressStatus.Message is not null)
             request.Message = progressStatus.Message;
         if (queueDepth is not null)

diff --git a/src/Serval/src/Serval.Client/Client.g.cs b/src/Serval/src/Serval.Client/Client.g.cs
@@ -7763,6 +7763,12 @@ public partial class TranslationBuild
         [Newtonsoft.Json.JsonProperty("percentCompleted", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
         public double? PercentCompleted { get; set; } = default!;
 
+        [Newtonsoft.Json.JsonProperty("fineTuneProgress", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
+        public double? FineTuneProgress { get; set; } = default!;
+
+        [Newtonsoft.Json.JsonProperty("inferenceProgress", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
+        public double? InferenceProgress { get; set; } = default!;
+
         [Newtonsoft.Json.JsonProperty("message", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
         public string? Message { get; set; } = default!;
 

diff --git a/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/platform.proto b/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/platform.proto
@@ -21,8 +21,10 @@ message UpdateBuildStatusRequest {
     string build_id = 1;
     int32 step = 2;
     optional double percent_completed = 3;
-    optional string message = 4;
-    optional int32 queue_depth = 5;
+    optional double fine_tune_progress = 4;
+    optional double inference_progress = 5;
+    optional string message = 6;
+    optional int32 queue_depth = 7;
 }
 
 message BuildStartedRequest {

diff --git a/src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs b/src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs
@@ -1,4 +1,4 @@
-namespace Serval.Translation.Contracts;
+namespace Serval.Translation.Contracts;
 
 public record TranslationBuildDto
 {
@@ -11,6 +11,9 @@ public record TranslationBuildDto
     public IReadOnlyList<PretranslateCorpusDto>? Pretranslate { get; init; }
     public required int Step { get; init; }
     public double? PercentCompleted { get; init; }
+    public double? FineTuneProgress { get; init; }
+    public double? InferenceProgress { get; init; }
+
     public string? Message { get; init; }
 
     public int? QueueDepth { get; init; }

diff --git a/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs b/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs
@@ -1596,6 +1596,8 @@ private TranslationBuildDto Map(Build source)
             Pretranslate = source.Pretranslate?.Select(s => Map(source.EngineRef, s)).ToList(),
             Step = source.Step,
             PercentCompleted = source.PercentCompleted,
+            FineTuneProgress = source.FineTuneProgress,
+            InferenceProgress = source.InferenceProgress,
             Message = source.Message,
             QueueDepth = source.QueueDepth,
             State = source.State,

diff --git a/src/Serval/src/Serval.Translation/Models/Build.cs b/src/Serval/src/Serval.Translation/Models/Build.cs
@@ -1,4 +1,4 @@
-namespace Serval.Translation.Models;
+namespace Serval.Translation.Models;
 
 public record Build : IInitializableEntity
 {
@@ -10,6 +10,8 @@ public record Build : IInitializableEntity
     public IReadOnlyList<PretranslateCorpus>? Pretranslate { get; init; }
     public int Step { get; init; }
     public double? PercentCompleted { get; init; }
+    public double? FineTuneProgress { get; init; }
+    public double? InferenceProgress { get; init; }
     public string? Message { get; init; }
     public int? QueueDepth { get; init; }
     public JobState State { get; init; } = JobState.Pending;

diff --git a/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs b/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs
@@ -218,6 +218,8 @@ await _dataAccessContext.WithTransactionAsync(
                         u.Set(b => b.Message, "Restarting")
                             .Set(b => b.Step, 0)
                             .Set(b => b.PercentCompleted, 0)
+                            .Set(b => b.FineTuneProgress, 0)
+                            .Set(b => b.InferenceProgress, 0)
                             .Set(b => b.State, JobState.Pending),
                     cancellationToken: ct
                 );
@@ -254,6 +256,20 @@ await _builds.UpdateAsync(
                         Math.Round(request.PercentCompleted, 4, MidpointRounding.AwayFromZero)
                     );
                 }
+                if (request.HasFineTuneProgress)
+                {
+                    u.Set(
+                        b => b.FineTuneProgress,
+                        Math.Round(request.FineTuneProgress, 4, MidpointRounding.AwayFromZero)
+                    );
+                }
+                if (request.HasInferenceProgress)
+                {
+                    u.Set(
+                        b => b.InferenceProgress,
+                        Math.Round(request.InferenceProgress, 4, MidpointRounding.AwayFromZero)
+                    );
+                }
                 if (request.HasMessage)
                     u.Set(b => b.Message, request.Message);
                 if (request.HasQueueDepth)

diff --git a/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs
@@ -83,12 +83,16 @@ await env.PlatformService.UpdateBuildStatus(
             {
                 BuildId = "b0",
                 QueueDepth = 1,
-                PercentCompleted = 0.5
+                PercentCompleted = 0.5,
+                FineTuneProgress = 0.25,
+                InferenceProgress = 0.3,
             },
             env.ServerCallContext
         );
         Assert.That(env.Builds.Get("b0").QueueDepth, Is.EqualTo(1));
         Assert.That(env.Builds.Get("b0").PercentCompleted, Is.EqualTo(0.5));
+        Assert.That(env.Builds.Get("b0").FineTuneProgress, Is.EqualTo(0.25));
+        Assert.That(env.Builds.Get("b0").InferenceProgress, Is.EqualTo(0.3));
     }
 
     [Test]