diff --git a/src/Serval.Translation/Controllers/TranslationEnginesController.cs b/src/Serval.Translation/Controllers/TranslationEnginesController.cs index 7001b331..3a09a127 100644 --- a/src/Serval.Translation/Controllers/TranslationEnginesController.cs +++ b/src/Serval.Translation/Controllers/TranslationEnginesController.cs @@ -1,4 +1,5 @@ using System.Net.Sockets; +using Serval.Translation.Utils; namespace Serval.Translation.Controllers; @@ -1075,11 +1076,9 @@ private static Build Map(Engine engine, TranslationBuildConfigDto source) } try { - var jsonSerializerOptions = new JsonSerializerOptions(); - jsonSerializerOptions.Converters.Add(new ObjectToInferredTypesConverter()); - build.Options = JsonSerializer.Deserialize>( + build.Options = Newtonsoft.Json.JsonConvert.DeserializeObject>( source.Options?.ToString() ?? "{}", - jsonSerializerOptions + new DictionaryJsonConverter() ); } catch (Exception e) diff --git a/src/Serval.Translation/Utils/DictionaryJsonConverter.cs b/src/Serval.Translation/Utils/DictionaryJsonConverter.cs new file mode 100644 index 00000000..226878d0 --- /dev/null +++ b/src/Serval.Translation/Utils/DictionaryJsonConverter.cs @@ -0,0 +1,167 @@ +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace Serval.Translation.Utils; + +// Credit to anish-patel post in https://stackoverflow.com/questions/11561597/deserialize-json-recursively-to-idictionarystring-object +public class DictionaryJsonConverter : JsonConverter +{ + public override void WriteJson(JsonWriter writer, object? value, Newtonsoft.Json.JsonSerializer serializer) + { + WriteValue(writer, value); + } + + private void WriteValue(JsonWriter writer, object? value) + { + if (value is null) + { + writer.WriteNull(); + return; + } + var t = JToken.FromObject(value); + switch (t.Type) + { + case JTokenType.Object: + WriteObject(writer, value); + break; + case JTokenType.Array: + WriteArray(writer, value); + break; + default: + writer.WriteValue(value); + break; + } + } + + private void WriteObject(JsonWriter writer, object value) + { + writer.WriteStartObject(); + var obj = + value as IDictionary + ?? throw new JsonSerializationException("Object must implement IDictionary"); + foreach (var kvp in obj) + { + writer.WritePropertyName(kvp.Key); + WriteValue(writer, kvp.Value); + } + writer.WriteEndObject(); + } + + private void WriteArray(JsonWriter writer, object value) + { + writer.WriteStartArray(); + var array = + value as IEnumerable + ?? throw new JsonSerializationException( + "Unexpected type when converting IDictionary to Array." + ); + foreach (var o in array) + { + WriteValue(writer, o); + } + writer.WriteEndArray(); + } + + public override object ReadJson( + JsonReader reader, + Type objectType, + object? existingValue, + Newtonsoft.Json.JsonSerializer serializer + ) + { + return ReadValue(reader); + } + + private object ReadValue(JsonReader reader) + { + while (reader.TokenType == JsonToken.Comment) + { + if (!reader.Read()) + throw new JsonSerializationException("Unexpected Token when converting IDictionary"); + } + + return reader.TokenType switch + { + JsonToken.StartObject => ReadObject(reader), + JsonToken.StartArray => ReadArray(reader), + JsonToken.Integer + or JsonToken.Float + or JsonToken.String + or JsonToken.Boolean + or JsonToken.Undefined + or JsonToken.Null + or JsonToken.Date + or JsonToken.Bytes + => reader.Value + ?? throw new JsonSerializationException( + "Unexpected token when converting to " + reader.TokenType.ToString() + ), + _ + => throw new JsonSerializationException( + string.Format("Unexpected token when converting IDictionary: {0}", reader.TokenType) + ), + }; + } + + private object ReadArray(JsonReader reader) + { + IList list = []; + + while (reader.Read()) + { + switch (reader.TokenType) + { + case JsonToken.Comment: + break; + default: + var v = ReadValue(reader); + + list.Add(v); + break; + case JsonToken.EndArray: + return list; + } + } + + throw new JsonSerializationException("Unexpected end when reading IDictionary"); + } + + private object ReadObject(JsonReader reader) + { + var obj = new Dictionary(); + + while (reader.Read()) + { + switch (reader.TokenType) + { + case JsonToken.PropertyName: + var propertyName = + reader.Value!.ToString() + ?? throw new JsonSerializationException( + "Unexpected Token when converting IDictionary" + ); + + if (!reader.Read()) + { + throw new JsonSerializationException("Unexpected end when reading IDictionary"); + } + + var v = ReadValue(reader); + + obj[propertyName] = v; + break; + case JsonToken.Comment: + break; + case JsonToken.EndObject: + return obj; + } + } + + throw new JsonSerializationException("Unexpected end when reading IDictionary"); + } + + public override bool CanConvert(Type objectType) + { + return typeof(IDictionary).IsAssignableFrom(objectType); + } +} diff --git a/tests/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs b/tests/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs index bc0f4cee..3e208868 100644 --- a/tests/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs +++ b/tests/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs @@ -64,7 +64,7 @@ public async Task SetUp() TargetLanguage = "en", Type = "Echo", Owner = "client1", - Corpora = new List() + Corpora = [] }; var e1 = new Engine { @@ -74,7 +74,7 @@ public async Task SetUp() TargetLanguage = "en", Type = "Echo", Owner = "client1", - Corpora = new List() + Corpora = [] }; var e2 = new Engine { @@ -84,7 +84,7 @@ public async Task SetUp() TargetLanguage = "en", Type = "Echo", Owner = "client2", - Corpora = new List() + Corpora = [] }; var be0 = new Engine { @@ -94,7 +94,7 @@ public async Task SetUp() TargetLanguage = "es", Type = "SMTTransfer", Owner = "client1", - Corpora = new List() + Corpora = [] }; var ce0 = new Engine { @@ -104,7 +104,7 @@ public async Task SetUp() TargetLanguage = "es", Type = "Nmt", Owner = "client1", - Corpora = new List() + Corpora = [] }; await _env.Engines.InsertAllAsync(new[] { e0, e1, e2, be0, ce0 }); @@ -740,7 +740,7 @@ public async Task GetAllPretranslationsAsync_Exists() CorpusRef = addedCorpus.Id, TextId = "all", EngineRef = ECHO_ENGINE1_ID, - Refs = new List { "ref1", "ref2" }, + Refs = ["ref1", "ref2"], Translation = "translation", ModelRevision = 1 }; @@ -799,7 +799,7 @@ public async Task GetAllPretranslationsAsync_TextIdExists() CorpusRef = addedCorpus.Id, TextId = "all", EngineRef = ECHO_ENGINE1_ID, - Refs = new List { "ref1", "ref2" }, + Refs = ["ref1", "ref2"], Translation = "translation", ModelRevision = 1 }; @@ -825,7 +825,7 @@ public async Task GetAllPretranslationsAsync_TextIdDoesNotExist() CorpusRef = addedCorpus.Id, TextId = "all", EngineRef = ECHO_ENGINE1_ID, - Refs = new List { "ref1", "ref2" }, + Refs = ["ref1", "ref2"], Translation = "translation", ModelRevision = 1 }; @@ -964,22 +964,19 @@ public async Task StartBuildForEngineByIdAsync(IEnumerable scope, int ex { case 201: TranslationCorpus addedCorpus = await client.AddCorpusAsync(engineId, TestCorpusConfig); - ptcc = new PretranslateCorpusConfig - { - CorpusId = addedCorpus.Id, - TextIds = new List { "all" } - }; - tcc = new() - { - CorpusId = addedCorpus.Id, - TextIds = new List { "all" } - }; + ptcc = new PretranslateCorpusConfig { CorpusId = addedCorpus.Id, TextIds = ["all"] }; + tcc = new() { CorpusId = addedCorpus.Id, TextIds = ["all"] }; tbc = new TranslationBuildConfig { - Pretranslate = new List { ptcc }, - TrainOn = new List { tcc }, - Options = - "{\"max_steps\":10, \"use_key_terms\":false, \"some_double\":10.5, \"some_string\":\"string\"}" + Pretranslate = [ptcc], + TrainOn = [tcc], + Options = """ + {"max_steps":10, + "use_key_terms":false, + "some_double":10.5, + "some_nested": {"more_nested": {"other_double":10.5}}, + "some_string":"string"} + """ }; TranslationBuild resultAfterStart; Assert.ThrowsAsync(async () => @@ -996,12 +993,8 @@ public async Task StartBuildForEngineByIdAsync(IEnumerable scope, int ex case 400: case 403: case 404: - ptcc = new PretranslateCorpusConfig - { - CorpusId = "cccccccccccccccccccccccc", - TextIds = new List { "all" } - }; - tbc = new TranslationBuildConfig { Pretranslate = new List { ptcc } }; + ptcc = new PretranslateCorpusConfig { CorpusId = "cccccccccccccccccccccccc", TextIds = ["all"] }; + tbc = new TranslationBuildConfig { Pretranslate = [ptcc] }; var ex = Assert.ThrowsAsync(async () => { await client.StartBuildAsync(engineId, tbc); @@ -1112,12 +1105,8 @@ public async Task TryToQueueMultipleBuildsPerSingleUser() var engineId = NMT_ENGINE1_ID; var expectedStatusCode = 409; TranslationCorpus addedCorpus = await client.AddCorpusAsync(engineId, TestCorpusConfigNonEcho); - var ptcc = new PretranslateCorpusConfig - { - CorpusId = addedCorpus.Id, - TextIds = new List { "all" } - }; - var tbc = new TranslationBuildConfig { Pretranslate = new List { ptcc } }; + var ptcc = new PretranslateCorpusConfig { CorpusId = addedCorpus.Id, TextIds = ["all"] }; + var tbc = new TranslationBuildConfig { Pretranslate = [ptcc] }; TranslationBuild build = await client.StartBuildAsync(engineId, tbc); var ex = Assert.ThrowsAsync(async () => { @@ -1174,7 +1163,7 @@ private static AsyncUnaryCall CreateAsyncUnaryCall(StatusC Task.FromException(new RpcException(status)), Task.FromResult(new Metadata()), () => status, - () => new Metadata(), + () => [], () => { } ); } @@ -1185,7 +1174,7 @@ private static AsyncUnaryCall CreateAsyncUnaryCall(TRespon Task.FromResult(response), Task.FromResult(new Metadata()), () => Status.DefaultSuccess, - () => new Metadata(), + () => [], () => { } ); } diff --git a/tests/Serval.E2ETests/ServalApiTests.cs b/tests/Serval.E2ETests/ServalApiTests.cs index 10e88f7d..c28c6bcd 100644 --- a/tests/Serval.E2ETests/ServalApiTests.cs +++ b/tests/Serval.E2ETests/ServalApiTests.cs @@ -116,6 +116,7 @@ public async Task NmtBatch() new TrainingCorpusConfig { CorpusId = cId1, TextIds = new string[] { "1JN.txt" } } }; var cId2 = await _helperClient.AddTextCorpusToEngine(engineId, new string[] { "3JN.txt" }, "es", "en", true); + _helperClient.TranslationBuildConfig.Options = """{"train_params":{"save_strategy": "yes"}}"""; await _helperClient.BuildEngine(engineId); await Task.Delay(1000); IList lTrans = await _helperClient.translationEnginesClient.GetAllPretranslationsAsync(