Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[TH2 5230] decodeComponentsToNestedMaps settings option #5

Merged
merged 4 commits into from
Aug 20, 2024
Merged

[TH2 5230] decodeComponentsToNestedMaps settings option #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1cef2c9
`decodeComponentsToNestedMaps` settings option added
lumber1000 Aug 19, 2024
bf80b53
validation for required fields in components improved
lumber1000 Aug 20, 2024
598ff23
Merge branch 'dev-ns' into TH2-5230
lumber1000 Aug 20, 2024
be5a92f
README.md
lumber1000 Aug 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ dictionary: fix_dictionary.xml
charset: US_ASCII
dirtyMode: false
decodeValuesToStrings: true
decodeComponentsToNestedMaps: true
```

#### beginString
Expand All @@ -32,7 +33,10 @@ default value: `US_ASCII`. Charset for reading and writing FIX fields.
default value: `false`. If `true`, processes all messages in dirty mode (generates warnings on invalid messages and continues processing). If `false`, only messages that contain the `encode-mode: dirty` property will be processed in dirty mode.

#### decodeValuesToStrings
default value: `true`. Decode all values to strings instead of typed values.
default value: `true`. If `true`, decodes all values to strings instead of typed values.

#### decodeComponentsToNestedMaps
default value: `true`. If `true`, decodes `components` to nested maps instead of unwrap component's map to message's main map.

## Release notes
### 0.1.0
Expand Down
85 changes: 52 additions & 33 deletions src/main/kotlin/com/exactpro/th2/codec/fixng/FixNgCodec.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,10 @@ class FixNgCodec(dictionary: IDictionaryStructure, settings: FixNgCodecSettings)
private val charset = settings.charset
private val isDirtyMode = settings.dirtyMode
private val isDecodeToStrings = settings.decodeValuesToStrings
private val isDecodeComponentsToNestedMaps = settings.decodeComponentsToNestedMaps

private val fieldsEncode = convertToFieldsByName(dictionary.fields, true)
private val fieldsDecode = convertToFieldsByTag(dictionary.fields)
private val fieldsEncode = convertToFieldsByName(dictionary.fields, true, emptyList(), true)
private val fieldsDecode = convertToFieldsByTag(dictionary.fields, emptyList())
private val messagesByTypeForDecode: Map<String, Message>
private val messagesByNameForEncode: Map<String, Message>

Expand Down Expand Up @@ -169,6 +170,7 @@ class FixNgCodec(dictionary: IDictionaryStructure, settings: FixNgCodecSettings)
private fun Field.decode(
source: ByteBuf,
target: MutableMap<String, Any>,
tagsSet: MutableSet<Int>,
value: String,
tag: Int,
isDirty: Boolean,
Expand All @@ -190,23 +192,25 @@ class FixNgCodec(dictionary: IDictionaryStructure, settings: FixNgCodecSettings)
}
}

// in dirty mode we'll use tag as field name if we have field duplication
val fieldName = if (isDirty && target.contains(name)) {
tag.toString()
} else {
name
if (!tagsSet.add(tag)) {
// we always handle tag duplication as error (isDirty = false)
handleError(false, context, "Duplicate $name field ($tag) with value: $value", value)
}

val previous = target.put(fieldName, decodedValue)
val targetMap = if (isDecodeComponentsToNestedMaps) {
@Suppress("UNCHECKED_CAST")
path.fold(target) { map, key -> map.computeIfAbsent(key) { mutableMapOf<String, Any>() } as MutableMap<String, Any> }
} else {
target
}

// but even in dirty mode we can't write field if it's duplicated more than once
// because we use Map and it cant contain duplicates
check(previous == null) { "Duplicate $name field ($tag) with value: $value (previous: $previous)" }
targetMap[name] = decodedValue
}

private val prereadHeaderFields = arrayOf("BeginString", "BodyLength", "MsgType")

private fun Message.decode(source: ByteBuf, bodyDef: Message, isDirty: Boolean, dictionaryFields: Map<Int, Field>, context: IReportingContext): MutableMap<String, Any> = mutableMapOf<String, Any>().also { map ->
val tagsSet: MutableSet<Int> = hashSetOf()
source.forEachField(charset, isDirty) { tag, value ->
val field = get(tag) ?: if (isDirty) {
when (this) {
Expand All @@ -221,19 +225,27 @@ class FixNgCodec(dictionary: IDictionaryStructure, settings: FixNgCodecSettings)
dictField
} else {
context.warning(DIRTY_MODE_WARNING_PREFIX + "Field does not exist in dictionary. Field tag: $tag. Field value: $value.")
Primitive(false, tag.toString(), String::class.java, emptySet(), tag)
Primitive(false, tag.toString(), emptyList(), String::class.java, emptySet(), tag)
}
} else {
// we reached next part of the message
return@forEachField false
}

field.decode(source, map, value, tag, isDirty, context)
field.decode(source, map, tagsSet, value, tag, isDirty, context)
return@forEachField true
}

for (field in fields.values) {
if (field.isRequired && !map.contains(field.name) && field.name !in prereadHeaderFields) {
if (!field.isRequired) continue

val tag = when (field) {
is Primitive -> field.tag
is Group -> field.counter
else -> error("Only `Primitive` and `Group` fields expected to be `required`")
}

if (!tagsSet.contains(tag) && field.name !in prereadHeaderFields) {
handleError(isDirty, context, "Required field missing. Field name: ${field.name}.")
}
}
Expand Down Expand Up @@ -299,7 +311,7 @@ class FixNgCodec(dictionary: IDictionaryStructure, settings: FixNgCodecSettings)
source.forEachField(charset, isDirty) { tag, value ->
val field = get(tag) ?: return@forEachField false

val group = if (tag == delimiter || !tags.add(tag) || map == null) {
val group = if (tag == delimiter || tags.contains(tag) || map == null) {
if (tag != delimiter) {
handleError(isDirty, context, "Field ${field.name} ($tag) appears before delimiter ($delimiter)")
}
Expand All @@ -313,7 +325,7 @@ class FixNgCodec(dictionary: IDictionaryStructure, settings: FixNgCodecSettings)
map ?: error("Group entry map can't be null.")
}

field.decode(source, group, value, tag, isDirty, context)
field.decode(source, group, tags, value, tag, isDirty, context)
return@forEachField true
}

Expand Down Expand Up @@ -432,11 +444,13 @@ class FixNgCodec(dictionary: IDictionaryStructure, settings: FixNgCodecSettings)
interface Field {
val isRequired: Boolean
val name: String
val path: List<String>
}

data class Primitive(
override val isRequired: Boolean,
override val name: String,
override val path: List<String>,
val primitiveType: Class<*>,
val values: Set<String>,
val tag: Int
Expand All @@ -463,13 +477,15 @@ class FixNgCodec(dictionary: IDictionaryStructure, settings: FixNgCodecSettings)
data class Message(
override val isRequired: Boolean,
override val name: String,
override val path: List<String>,
val type: String,
override val fields: Map<String, Field>,
) : Field, FieldMap()

data class Group(
override val isRequired: Boolean,
override val name: String,
override val path: List<String>,
val counter: Int,
val delimiter: Int,
override val fields: Map<String, Field>,
Expand Down Expand Up @@ -536,59 +552,62 @@ class FixNgCodec(dictionary: IDictionaryStructure, settings: FixNgCodecSettings)
private val IFieldStructure.tag: Int
get() = StructureUtils.getAttributeValue(this, "tag")

private fun IFieldStructure.toPrimitive(): Primitive = Primitive(
isRequired,
private fun IFieldStructure.toPrimitive(path: List<String>, isRequiredParent: Boolean): Primitive = Primitive(
isRequiredParent && isRequired,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note: the field is required if parent block is presented in message.

name,
path,
javaTypeToClass.getValue(javaType),
values.values.map<IAttributeStructure, String> { it.getCastValue<Any>().toString() }.toSet(),
tag
)

private fun convertToFieldsByName(fields: Map<String, IFieldStructure>, isForEncode: Boolean): Map<String, Field> = linkedMapOf<String, Field>().apply {
private fun convertToFieldsByName(fields: Map<String, IFieldStructure>, isForEncode: Boolean, path: List<String>, isRequiredParent: Boolean): Map<String, Field> = linkedMapOf<String, Field>().apply {
fields.forEach { (name, field) ->
when {
field !is IMessageStructure -> this[name] = field.toPrimitive()
field.isGroup -> this[name] = field.toGroup(isForEncode)
field !is IMessageStructure -> this[name] = field.toPrimitive(path, isForEncode || isRequiredParent)
field.isGroup -> this[name] = field.toGroup(isForEncode, path, isForEncode || isRequiredParent)
field.isComponent -> if (isForEncode) {
this[name] = field.toMessage(true)
this[name] = field.toMessage(true, path + name)
} else {
this += convertToFieldsByName(field.fields, false)
this += convertToFieldsByName(field.fields, false, path + name, isRequiredParent && field.isRequired)
}
}
}
}

private fun convertToFieldsByTag(fields: Map<String, IFieldStructure>): Map<Int, Field> = linkedMapOf<Int, Field>().apply {
private fun convertToFieldsByTag(fields: Map<String, IFieldStructure>, path: List<String>): Map<Int, Field> = linkedMapOf<Int, Field>().apply {
fields.values.forEach { field ->
when {
field !is IMessageStructure -> this[field.tag] = field.toPrimitive()
field.isGroup -> this[field.tag] = field.toGroup(false)
field.isComponent -> this += convertToFieldsByTag(field.fields)
field !is IMessageStructure -> this[field.tag] = field.toPrimitive(path, true)
field.isGroup -> this[field.tag] = field.toGroup(false, path, true)
field.isComponent -> this += convertToFieldsByTag(field.fields, path + field.name)
}
}
}

private fun IMessageStructure.toMessage(isForEncode: Boolean): Message = Message(
private fun IMessageStructure.toMessage(isForEncode: Boolean, path: List<String>): Message = Message(
name = name,
type = StructureUtils.getAttributeValue(this, FIELD_MESSAGE_TYPE) ?: name,
fields = convertToFieldsByName(this.fields, isForEncode),
fields = convertToFieldsByName(this.fields, isForEncode, path, !isComponent || isRequired),
path = path,
isRequired = isRequired
)

private fun getFirstTag(message: IMessageStructure): Int = message.fields.values.first().let {
if (it is IMessageStructure && it.isComponent) getFirstTag(it) else it.tag
}

private fun IMessageStructure.toGroup(isForEncode: Boolean): Group = Group(
private fun IMessageStructure.toGroup(isForEncode: Boolean, path: List<String>, isRequiredParent: Boolean): Group = Group(
name = name,
counter = tag,
delimiter = getFirstTag(this),
fields = convertToFieldsByName(this.fields, isForEncode),
isRequired = isRequired
fields = convertToFieldsByName(this.fields, isForEncode, emptyList(), true),
path = path,
isRequired = isRequiredParent && isRequired
)

fun IDictionaryStructure.toMessages(isForEncode: Boolean): List<Message> = messages.values
.filterNot { it.isGroup || it.isComponent }
.map { it.toMessage(isForEncode) }
.map { it.toMessage(isForEncode, emptyList()) }
}
}
1 change: 1 addition & 0 deletions src/main/kotlin/com/exactpro/th2/codec/fixng/FixNgCodecSettings.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,4 +25,5 @@ data class FixNgCodecSettings(
val charset: Charset = Charsets.US_ASCII,
val dirtyMode: Boolean = false,
val decodeValuesToStrings: Boolean = true,
val decodeComponentsToNestedMaps: Boolean = true
) : IPipelineCodecSettings
Loading