Cleanup builds & engines

src/Serval/src/Serval.Translation/Configuration/IServalBuilderExtensions.cs

@@ -25,6 +25,9 @@ public static IServalBuilder AddTranslation(
         builder.Services.AddScoped<IPretranslationService, PretranslationService>();
         builder.Services.AddScoped<IEngineService, EngineService>();
 
+        builder.Services.AddSingleton<EngineCleanupService>();
+        builder.Services.AddSingleton<BuildCleanupService>();
+
         var translationOptions = new TranslationOptions();
         builder.Configuration?.GetSection(TranslationOptions.Key).Bind(translationOptions);
         if (configure is not null)

src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs

@@ -1296,7 +1296,8 @@ private Engine Map(TranslationEngineConfigDto source)
             Type = source.Type.ToPascalCase(),
             Owner = Owner,
             Corpora = [],
-            IsModelPersisted = source.IsModelPersisted
+            IsModelPersisted = source.IsModelPersisted,
+            SuccessfullyCreated = false
         };
     }
 
@@ -1308,7 +1309,8 @@ private static Build Map(Engine engine, TranslationBuildConfigDto source)
             Name = source.Name,
             Pretranslate = Map(engine, source.Pretranslate),
             TrainOn = Map(engine, source.TrainOn),
-            Options = Map(source.Options)
+            Options = Map(source.Options),
+            SuccessfullyStarted = false
         };
     }
 

src/Serval/src/Serval.Translation/Models/Build.cs

@@ -15,4 +15,5 @@ public record Build : IEntity
     public JobState State { get; init; } = JobState.Pending;
     public DateTime? DateFinished { get; init; }
     public IReadOnlyDictionary<string, object>? Options { get; init; }
+    public bool SuccessfullyStarted { get; init; }
 }

src/Serval/src/Serval.Translation/Models/Engine.cs

@@ -16,4 +16,5 @@ public record Engine : IOwnedEntity
     public int ModelRevision { get; init; }
     public double Confidence { get; init; }
     public int CorpusSize { get; init; }
+    public bool SuccessfullyCreated { get; init; }
 }

src/Serval/src/Serval.Translation/Services/BuildCleanupService.cs

@@ -0,0 +1,42 @@
+using Microsoft.Extensions.DependencyInjection;
+using SIL.ServiceToolkit.Services;
+
+namespace Serval.Translation.Services;
+
+public class BuildCleanupService(
+    IServiceProvider services,
+    ILogger<BuildCleanupService> logger,
+    TimeSpan? timeout = null
+) : RecurrentTask("Build Cleanup Service", services, RefreshPeriod, logger)
+{
+    private readonly ILogger<BuildCleanupService> _logger = logger;
+    private readonly TimeSpan _timeout = timeout ?? TimeSpan.FromMinutes(2);
+    private static readonly TimeSpan RefreshPeriod = TimeSpan.FromDays(1);
+
+    protected override async Task DoWorkAsync(IServiceScope scope, CancellationToken cancellationToken)
+    {
+        _logger.LogInformation("Running build cleanup job");
+        var builds = scope.ServiceProvider.GetRequiredService<IRepository<Build>>();
+        await CheckBuildsAsync(builds, cancellationToken);
+    }
+
+    public async Task CheckBuildsAsync(IRepository<Build> builds, CancellationToken cancellationToken)
+    {
+        IReadOnlyList<Build> allBuilds = await builds.GetAllAsync(cancellationToken);
+        IEnumerable<Build> notStartedBuilds = allBuilds.Where(b => !b.SuccessfullyStarted);
+        await Task.Delay(_timeout, cancellationToken); //Make sure the builds are not midway through starting
+        foreach (
+            Build build in await builds.GetAllAsync(
+                b => notStartedBuilds.Select(c => c.Id).Contains(b.Id),
+                cancellationToken
+            )
+        )
+        {
+            if (!build.SuccessfullyStarted)
+            {
+                _logger.LogInformation("Deleting build {id} because it was never successfully started", build.Id);
+                await builds.DeleteAsync(build, cancellationToken);
+            }
+        }
+    }
+}

src/Serval/src/Serval.Translation/Services/EngineCleanupService.cs

@@ -0,0 +1,47 @@
+using Microsoft.Extensions.DependencyInjection;
+using SIL.ServiceToolkit.Services;
+
+namespace Serval.Translation.Services;
+
+public class EngineCleanupService(
+    IServiceProvider services,
+    ILogger<EngineCleanupService> logger,
+    TimeSpan? timeout = null
+) : RecurrentTask("Engine Cleanup Service", services, RefreshPeriod, logger)
+{
+    private readonly ILogger<EngineCleanupService> _logger = logger;
+    private readonly TimeSpan _timeout = timeout ?? TimeSpan.FromMinutes(2);
+    private static readonly TimeSpan RefreshPeriod = TimeSpan.FromDays(1);
+
+    protected override async Task DoWorkAsync(IServiceScope scope, CancellationToken cancellationToken)
+    {
+        _logger.LogInformation("Running engine cleanup job");
+        var engines = scope.ServiceProvider.GetRequiredService<IRepository<Engine>>();
+        var engineService = scope.ServiceProvider.GetRequiredService<EngineService>();
+        await CheckEnginesAsync(engines, engineService, cancellationToken);
+    }
+
+    public async Task CheckEnginesAsync(
+        IRepository<Engine> engines,
+        EngineService engineService,
+        CancellationToken cancellationToken
+    )
+    {
+        IReadOnlyList<Engine> allEngines = await engines.GetAllAsync(cancellationToken);
+        IEnumerable<Engine> notCreatedEngines = allEngines.Where(e => !e.SuccessfullyCreated);
+        await Task.Delay(_timeout, cancellationToken); //Make sure the engines are not midway through being created
+        foreach (
+            Engine engine in await engines.GetAllAsync(
+                e => notCreatedEngines.Select(f => f.Id).Contains(e.Id),
+                cancellationToken
+            )
+        )
+        {
+            if (!engine.SuccessfullyCreated)
+            {
+                _logger.LogInformation("Deleting engine {id} because it was never successfully created", engine.Id);
+                await engineService.DeleteAsync(engine.Id, cancellationToken);
+            }
+        }
+    }
+}

src/Serval/src/Serval.Translation/Services/EngineService.cs

@@ -146,6 +146,11 @@ public override async Task<Engine> CreateAsync(Engine engine, CancellationToken
             {
                 IsModelPersisted = createResponse.IsModelPersisted
             };
+            await Entities.UpdateAsync(
+                engine,
+                u => u.Set(e => e.SuccessfullyCreated, true),
+                cancellationToken: cancellationToken
+            );
         }
         catch (RpcException rpcex)
         {
@@ -292,6 +297,11 @@ public async Task StartBuildAsync(Build build, CancellationToken cancellationTok
                 _logger.LogInformation("{request}", JsonSerializer.Serialize(request));
             }
             await client.StartBuildAsync(request, cancellationToken: cancellationToken);
+            await _builds.UpdateAsync(
+                build.Id,
+                u => u.Set(e => e.SuccessfullyStarted, true),
+                cancellationToken: cancellationToken
+            );
         }
         catch
         {

src/Serval/test/Serval.Translation.Tests/Services/BuildCleanupServiceTests.cs

@@ -0,0 +1,54 @@
+namespace Serval.Translation.Services;
+
+[TestFixture]
+public class BuildCleanupServiceTests
+{
+    [Test]
+    public async Task CleanupAsync()
+    {
+        TestEnvironment env = new();
+        Assert.That(env.Builds.Count, Is.EqualTo(2));
+        await env.CheckBuildsAsync();
+        Assert.That(env.Builds.Count, Is.EqualTo(1));
+        Assert.That((await env.Builds.GetAllAsync())[0].Id, Is.EqualTo("build2"));
+    }
+
+    private class TestEnvironment
+    {
+        public MemoryRepository<Build> Builds { get; }
+
+        public TestEnvironment()
+        {
+            Builds = new MemoryRepository<Build>();
+            Builds.Add(
+                new Build
+                {
+                    Id = "build1",
+                    EngineRef = "engine1",
+                    SuccessfullyStarted = false
+                }
+            );
+            Builds.Add(
+                new Build
+                {
+                    Id = "build2",
+                    EngineRef = "engine2",
+                    SuccessfullyStarted = true
+                }
+            );
+
+            Service = new BuildCleanupService(
+                Substitute.For<IServiceProvider>(),
+                Substitute.For<ILogger<BuildCleanupService>>(),
+                TimeSpan.Zero
+            );
+        }
+
+        public BuildCleanupService Service { get; }
+
+        public async Task CheckBuildsAsync()
+        {
+            await Service.CheckBuildsAsync(Builds, CancellationToken.None);
+        }
+    }
+}

src/Serval/test/Serval.Translation.Tests/Services/EngineCleanupServiceTests.cs

@@ -0,0 +1,97 @@
+using Google.Protobuf.WellKnownTypes;
+using MassTransit.Mediator;
+using Serval.Translation.V1;
+
+namespace Serval.Translation.Services;
+
+[TestFixture]
+public class EngineCleanupServiceTests
+{
+    [Test]
+    public async Task CleanupAsync()
+    {
+        TestEnvironment env = new();
+        Assert.That(env.Engines.Count, Is.EqualTo(2));
+        await env.CheckEnginesAsync();
+        Assert.That(env.Engines.Count, Is.EqualTo(1));
+        Assert.That((await env.Engines.GetAllAsync())[0].Id, Is.EqualTo("engine2"));
+    }
+
+    private class TestEnvironment
+    {
+        public MemoryRepository<Engine> Engines { get; }
+
+        public TestEnvironment()
+        {
+            Engines = new MemoryRepository<Engine>();
+            Engines.Add(
+                new Engine
+                {
+                    Id = "engine1",
+                    SourceLanguage = "en",
+                    TargetLanguage = "es",
+                    Type = "Nmt",
+                    Owner = "client1",
+                    SuccessfullyCreated = false
+                }
+            );
+            Engines.Add(
+                new Engine
+                {
+                    Id = "engine2",
+                    SourceLanguage = "en",
+                    TargetLanguage = "es",
+                    Type = "Nmt",
+                    Owner = "client1",
+                    SuccessfullyCreated = true
+                }
+            );
+
+            Service = new EngineCleanupService(
+                Substitute.For<IServiceProvider>(),
+                Substitute.For<ILogger<EngineCleanupService>>(),
+                TimeSpan.Zero
+            );
+
+            var translationServiceClient = Substitute.For<TranslationEngineApi.TranslationEngineApiClient>();
+            translationServiceClient.DeleteAsync(Arg.Any<DeleteRequest>()).Returns(CreateAsyncUnaryCall(new Empty()));
+
+            GrpcClientFactory grpcClientFactory = Substitute.For<GrpcClientFactory>();
+            grpcClientFactory
+                .CreateClient<TranslationEngineApi.TranslationEngineApiClient>("Nmt")
+                .Returns(translationServiceClient);
+
+            _engineService = new EngineService(
+                Engines,
+                new MemoryRepository<Build>(),
+                new MemoryRepository<Pretranslation>(),
+                Substitute.For<IScopedMediator>(),
+                grpcClientFactory,
+                Substitute.For<IOptionsMonitor<DataFileOptions>>(),
+                new MemoryDataAccessContext(),
+                new LoggerFactory(),
+                Substitute.For<IScriptureDataFileService>()
+            );
+        }
+
+        public EngineCleanupService Service { get; }
+
+        private readonly EngineService _engineService;
+
+        public async Task CheckEnginesAsync()
+        {
+            await Service.CheckEnginesAsync(Engines, _engineService, CancellationToken.None);
+        }
+
+        private static AsyncUnaryCall<TResponse> CreateAsyncUnaryCall<TResponse>(TResponse response)
+        {
+            return new AsyncUnaryCall<TResponse>(
+                Task.FromResult(response),
+                Task.FromResult(new Metadata()),
+                () => Status.DefaultSuccess,
+                () => new Metadata(),
+                () => { }
+            );
+        }
+    }
+}