Skip to content

Commit

Permalink
Fix nested options the right way.
Browse files Browse the repository at this point in the history
  • Loading branch information
johnml1135 committed Jan 26, 2024
1 parent 41348df commit de81664
Show file tree
Hide file tree
Showing 4 changed files with 196 additions and 40 deletions.
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System.Net.Sockets;
using Serval.Translation.Utils;

namespace Serval.Translation.Controllers;

Expand Down Expand Up @@ -1075,11 +1076,9 @@ private static Build Map(Engine engine, TranslationBuildConfigDto source)
}
try
{
var jsonSerializerOptions = new JsonSerializerOptions();
jsonSerializerOptions.Converters.Add(new ObjectToInferredTypesConverter());
build.Options = JsonSerializer.Deserialize<IDictionary<string, object>>(
build.Options = Newtonsoft.Json.JsonConvert.DeserializeObject<IDictionary<string, object>>(
source.Options?.ToString() ?? "{}",
jsonSerializerOptions
new DictionaryJsonConverter()
);
}
catch (Exception e)
Expand Down
167 changes: 167 additions & 0 deletions src/Serval.Translation/Utils/DictionaryJsonConverter.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;

namespace Serval.Translation.Utils;

// Credit to anish-patel post in https://stackoverflow.com/questions/11561597/deserialize-json-recursively-to-idictionarystring-object
public class DictionaryJsonConverter : JsonConverter
{
public override void WriteJson(JsonWriter writer, object? value, Newtonsoft.Json.JsonSerializer serializer)
{
WriteValue(writer, value);
}

private void WriteValue(JsonWriter writer, object? value)
{
if (value is null)
{
writer.WriteNull();
return;
}
var t = JToken.FromObject(value);
switch (t.Type)
{
case JTokenType.Object:
WriteObject(writer, value);
break;
case JTokenType.Array:
WriteArray(writer, value);
break;
default:
writer.WriteValue(value);
break;
}
}

private void WriteObject(JsonWriter writer, object value)
{
writer.WriteStartObject();
var obj =
value as IDictionary<string, object>
?? throw new JsonSerializationException("Object must implement IDictionary<string, object>");
foreach (var kvp in obj)
{
writer.WritePropertyName(kvp.Key);
WriteValue(writer, kvp.Value);
}
writer.WriteEndObject();
}

private void WriteArray(JsonWriter writer, object value)
{
writer.WriteStartArray();
var array =
value as IEnumerable<object>
?? throw new JsonSerializationException(
"Unexpected type when converting IDictionary<string, object> to Array."
);
foreach (var o in array)
{
WriteValue(writer, o);
}
writer.WriteEndArray();
}

public override object ReadJson(
JsonReader reader,
Type objectType,
object? existingValue,
Newtonsoft.Json.JsonSerializer serializer
)
{
return ReadValue(reader);
}

private object ReadValue(JsonReader reader)
{
while (reader.TokenType == JsonToken.Comment)
{
if (!reader.Read())
throw new JsonSerializationException("Unexpected Token when converting IDictionary<string, object>");
}

return reader.TokenType switch
{
JsonToken.StartObject => ReadObject(reader),
JsonToken.StartArray => ReadArray(reader),
JsonToken.Integer
or JsonToken.Float
or JsonToken.String
or JsonToken.Boolean
or JsonToken.Undefined
or JsonToken.Null
or JsonToken.Date
or JsonToken.Bytes
=> reader.Value
?? throw new JsonSerializationException(
"Unexpected token when converting to " + reader.TokenType.ToString()
),
_
=> throw new JsonSerializationException(
string.Format("Unexpected token when converting IDictionary<string, object>: {0}", reader.TokenType)
),
};
}

private object ReadArray(JsonReader reader)
{
IList<object> list = [];

while (reader.Read())
{
switch (reader.TokenType)
{
case JsonToken.Comment:
break;
default:
var v = ReadValue(reader);

list.Add(v);
break;
case JsonToken.EndArray:
return list;
}
}

throw new JsonSerializationException("Unexpected end when reading IDictionary<string, object>");
}

private object ReadObject(JsonReader reader)
{
var obj = new Dictionary<string, object>();

while (reader.Read())
{
switch (reader.TokenType)
{
case JsonToken.PropertyName:
var propertyName =
reader.Value!.ToString()
?? throw new JsonSerializationException(
"Unexpected Token when converting IDictionary<string, object>"
);

if (!reader.Read())
{
throw new JsonSerializationException("Unexpected end when reading IDictionary<string, object>");
}

var v = ReadValue(reader);

obj[propertyName] = v;
break;
case JsonToken.Comment:
break;
case JsonToken.EndObject:
return obj;
}
}

throw new JsonSerializationException("Unexpected end when reading IDictionary<string, object>");
}

public override bool CanConvert(Type objectType)
{
return typeof(IDictionary<string, object>).IsAssignableFrom(objectType);
}
}
61 changes: 25 additions & 36 deletions tests/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ public async Task SetUp()
TargetLanguage = "en",
Type = "Echo",
Owner = "client1",
Corpora = new List<Translation.Models.Corpus>()
Corpora = []
};
var e1 = new Engine
{
Expand All @@ -74,7 +74,7 @@ public async Task SetUp()
TargetLanguage = "en",
Type = "Echo",
Owner = "client1",
Corpora = new List<Translation.Models.Corpus>()
Corpora = []
};
var e2 = new Engine
{
Expand All @@ -84,7 +84,7 @@ public async Task SetUp()
TargetLanguage = "en",
Type = "Echo",
Owner = "client2",
Corpora = new List<Translation.Models.Corpus>()
Corpora = []
};
var be0 = new Engine
{
Expand All @@ -94,7 +94,7 @@ public async Task SetUp()
TargetLanguage = "es",
Type = "SMTTransfer",
Owner = "client1",
Corpora = new List<Translation.Models.Corpus>()
Corpora = []
};
var ce0 = new Engine
{
Expand All @@ -104,7 +104,7 @@ public async Task SetUp()
TargetLanguage = "es",
Type = "Nmt",
Owner = "client1",
Corpora = new List<Translation.Models.Corpus>()
Corpora = []
};

await _env.Engines.InsertAllAsync(new[] { e0, e1, e2, be0, ce0 });
Expand Down Expand Up @@ -740,7 +740,7 @@ public async Task GetAllPretranslationsAsync_Exists()
CorpusRef = addedCorpus.Id,
TextId = "all",
EngineRef = ECHO_ENGINE1_ID,
Refs = new List<string> { "ref1", "ref2" },
Refs = ["ref1", "ref2"],
Translation = "translation",
ModelRevision = 1
};
Expand Down Expand Up @@ -799,7 +799,7 @@ public async Task GetAllPretranslationsAsync_TextIdExists()
CorpusRef = addedCorpus.Id,
TextId = "all",
EngineRef = ECHO_ENGINE1_ID,
Refs = new List<string> { "ref1", "ref2" },
Refs = ["ref1", "ref2"],
Translation = "translation",
ModelRevision = 1
};
Expand All @@ -825,7 +825,7 @@ public async Task GetAllPretranslationsAsync_TextIdDoesNotExist()
CorpusRef = addedCorpus.Id,
TextId = "all",
EngineRef = ECHO_ENGINE1_ID,
Refs = new List<string> { "ref1", "ref2" },
Refs = ["ref1", "ref2"],
Translation = "translation",
ModelRevision = 1
};
Expand Down Expand Up @@ -964,22 +964,19 @@ public async Task StartBuildForEngineByIdAsync(IEnumerable<string> scope, int ex
{
case 201:
TranslationCorpus addedCorpus = await client.AddCorpusAsync(engineId, TestCorpusConfig);
ptcc = new PretranslateCorpusConfig
{
CorpusId = addedCorpus.Id,
TextIds = new List<string> { "all" }
};
tcc = new()
{
CorpusId = addedCorpus.Id,
TextIds = new List<string> { "all" }
};
ptcc = new PretranslateCorpusConfig { CorpusId = addedCorpus.Id, TextIds = ["all"] };
tcc = new() { CorpusId = addedCorpus.Id, TextIds = ["all"] };
tbc = new TranslationBuildConfig
{
Pretranslate = new List<PretranslateCorpusConfig> { ptcc },
TrainOn = new List<TrainingCorpusConfig> { tcc },
Options =
"{\"max_steps\":10, \"use_key_terms\":false, \"some_double\":10.5, \"some_string\":\"string\"}"
Pretranslate = [ptcc],
TrainOn = [tcc],
Options = """
{"max_steps":10,
"use_key_terms":false,
"some_double":10.5,
"some_nested": {"more_nested": {"other_double":10.5}},
"some_string":"string"}
"""
};
TranslationBuild resultAfterStart;
Assert.ThrowsAsync<ServalApiException>(async () =>
Expand All @@ -996,12 +993,8 @@ public async Task StartBuildForEngineByIdAsync(IEnumerable<string> scope, int ex
case 400:
case 403:
case 404:
ptcc = new PretranslateCorpusConfig
{
CorpusId = "cccccccccccccccccccccccc",
TextIds = new List<string> { "all" }
};
tbc = new TranslationBuildConfig { Pretranslate = new List<PretranslateCorpusConfig> { ptcc } };
ptcc = new PretranslateCorpusConfig { CorpusId = "cccccccccccccccccccccccc", TextIds = ["all"] };
tbc = new TranslationBuildConfig { Pretranslate = [ptcc] };
var ex = Assert.ThrowsAsync<ServalApiException>(async () =>
{
await client.StartBuildAsync(engineId, tbc);
Expand Down Expand Up @@ -1112,12 +1105,8 @@ public async Task TryToQueueMultipleBuildsPerSingleUser()
var engineId = NMT_ENGINE1_ID;
var expectedStatusCode = 409;
TranslationCorpus addedCorpus = await client.AddCorpusAsync(engineId, TestCorpusConfigNonEcho);
var ptcc = new PretranslateCorpusConfig
{
CorpusId = addedCorpus.Id,
TextIds = new List<string> { "all" }
};
var tbc = new TranslationBuildConfig { Pretranslate = new List<PretranslateCorpusConfig> { ptcc } };
var ptcc = new PretranslateCorpusConfig { CorpusId = addedCorpus.Id, TextIds = ["all"] };
var tbc = new TranslationBuildConfig { Pretranslate = [ptcc] };
TranslationBuild build = await client.StartBuildAsync(engineId, tbc);
var ex = Assert.ThrowsAsync<ServalApiException>(async () =>
{
Expand Down Expand Up @@ -1174,7 +1163,7 @@ private static AsyncUnaryCall<TResponse> CreateAsyncUnaryCall<TResponse>(StatusC
Task.FromException<TResponse>(new RpcException(status)),
Task.FromResult(new Metadata()),
() => status,
() => new Metadata(),
() => [],
() => { }
);
}
Expand All @@ -1185,7 +1174,7 @@ private static AsyncUnaryCall<TResponse> CreateAsyncUnaryCall<TResponse>(TRespon
Task.FromResult(response),
Task.FromResult(new Metadata()),
() => Status.DefaultSuccess,
() => new Metadata(),
() => [],
() => { }
);
}
Expand Down
1 change: 1 addition & 0 deletions tests/Serval.E2ETests/ServalApiTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ public async Task NmtBatch()
new TrainingCorpusConfig { CorpusId = cId1, TextIds = new string[] { "1JN.txt" } }
};
var cId2 = await _helperClient.AddTextCorpusToEngine(engineId, new string[] { "3JN.txt" }, "es", "en", true);
_helperClient.TranslationBuildConfig.Options = """{"train_params":{"save_strategy": "yes"}}""";
await _helperClient.BuildEngine(engineId);
await Task.Delay(1000);
IList<Pretranslation> lTrans = await _helperClient.translationEnginesClient.GetAllPretranslationsAsync(
Expand Down

0 comments on commit de81664

Please sign in to comment.