diff --git a/README.md b/README.md index 083f5c6..6ec9425 100644 --- a/README.md +++ b/README.md @@ -59,12 +59,14 @@ This operation isn't support. Oracle log miner codec (transformer) has the following parameters: ```yaml -columnPrefix: 'th2_' -saveColumns: [ OPERATION, SQL_REDO, ROW_ID, TIMESTAMP, TABLE_NAME ] +column-prefix: 'th2_' +save-columns: [ OPERATION, SQL_REDO, ROW_ID, TIMESTAMP, TABLE_NAME ] ``` -**columnPrefix** - prefix for parsed columns. -**saveColumns** - set of column names to copy from source message. +**truncate-update-query-from-where-clause** - if true, codec truncates the tail of UPDATE query starting from the WHERE clause before deep parsing. +This operation improve performance without negative impact, because codec extracts data from the SET clause only. +**column-prefix** - prefix for parsed columns. +**save-columns** - set of column names to copy from source message. All columns which log miner allow to select are described in the [document](https://docs.oracle.com/en/database/oracle/oracle-database/19/refrn/V-LOGMNR_CONTENTS.html#GUID-B9196942-07BF-4935-B603-FA875064F5C3) ## Full configuration example @@ -87,6 +89,7 @@ spec: disableProtocolCheck: true codecSettings: + truncate-update-query-from-where-clause: true column-prefix: th2_ save-columns: - OPERATION @@ -139,7 +142,8 @@ spec: ## Release notes ### 0.1.0 -+ Migrated to ANTLR 4 approach for parsing Oracle SQL queries. ++ Migrated to ANTLR 4 approach for parsing Oracle SQL queries. ++ Added `truncate-update-query-from-where-clause` option temporary. ### 0.0.2 + Publish warning event with details about internal exception. diff --git a/src/main/kotlin/com/exactpro/th2/codec/oracle/logminer/LogMinerTransformer.kt b/src/main/kotlin/com/exactpro/th2/codec/oracle/logminer/LogMinerTransformer.kt index 8ec5cd6..086be8d 100644 --- a/src/main/kotlin/com/exactpro/th2/codec/oracle/logminer/LogMinerTransformer.kt +++ b/src/main/kotlin/com/exactpro/th2/codec/oracle/logminer/LogMinerTransformer.kt @@ -79,7 +79,12 @@ class LogMinerTransformer(private val config: LogMinerConfiguration) : IPipeline "UPDATE" -> { message.toBuilderWithoutBody().apply { bodyBuilder().apply { - val lexer = PlSqlLexer(CharStreams.fromString(sqlRedo)) + val preparedQuery = if (config.truncateUpdateQueryFromWhereClause) { + truncateFromWhereClause(sqlRedo) + } else { + sqlRedo + } + val lexer = PlSqlLexer(CharStreams.fromString(preparedQuery)) val tokens = CommonTokenStream(lexer) val parser = PlSqlParser(tokens) val walker = ParseTreeWalker() @@ -554,6 +559,7 @@ class LogMinerTransformer(private val config: LogMinerConfiguration) : IPipeline private const val LOG_MINER_ROW_ID_COLUMN = "ROW_ID" private const val LOG_MINER_TIMESTAMP_COLUMN = "TIMESTAMP" private const val LOG_MINER_TABLE_NAME_COLUMN = "TABLE_NAME" + private const val WHERE_CLAUSE = "WHERE" internal val REQUIRED_COLUMNS: Set = hashSetOf( LOG_MINER_OPERATION_COLUMN, @@ -563,6 +569,15 @@ class LogMinerTransformer(private val config: LogMinerConfiguration) : IPipeline LOG_MINER_TABLE_NAME_COLUMN, ) + internal fun truncateFromWhereClause(query: String): String { + val whereIndex = query.indexOf(WHERE_CLAUSE, ignoreCase = true) + return if (whereIndex == -1) { + query + } else { + "${query.substring(0, whereIndex)};" + } + } + internal fun ParsedMessage.toBuilderWithoutBody() = ParsedMessage.builder().apply { setId(this@toBuilderWithoutBody.id) this@toBuilderWithoutBody.eventId?.let(this::setEventId) diff --git a/src/main/kotlin/com/exactpro/th2/codec/oracle/logminer/cfg/LogMinerConfiguration.kt b/src/main/kotlin/com/exactpro/th2/codec/oracle/logminer/cfg/LogMinerConfiguration.kt index 4b92e40..b687bde 100644 --- a/src/main/kotlin/com/exactpro/th2/codec/oracle/logminer/cfg/LogMinerConfiguration.kt +++ b/src/main/kotlin/com/exactpro/th2/codec/oracle/logminer/cfg/LogMinerConfiguration.kt @@ -29,4 +29,8 @@ class LogMinerConfiguration : IPipelineCodecSettings { @JsonProperty("save-columns") @JsonPropertyDescription("Codec saves: extracted columns and this set") var saveColumns: Set = REQUIRED_COLUMNS + + @JsonProperty("truncate-update-query-from-where-clause") + @JsonPropertyDescription("Codec truncates the tail of UPDATE query starting from the WHERE clause") + var truncateUpdateQueryFromWhereClause: Boolean = true } diff --git a/src/test/kotlin/com/exactpro/th2/codec/oracle/logminer/LogMinerTransformerTest.kt b/src/test/kotlin/com/exactpro/th2/codec/oracle/logminer/LogMinerTransformerTest.kt index abad298..9f60943 100644 --- a/src/test/kotlin/com/exactpro/th2/codec/oracle/logminer/LogMinerTransformerTest.kt +++ b/src/test/kotlin/com/exactpro/th2/codec/oracle/logminer/LogMinerTransformerTest.kt @@ -19,6 +19,7 @@ package com.exactpro.th2.codec.oracle.logminer import PlSqlLexer import PlSqlParser import com.exactpro.th2.codec.api.IReportingContext +import com.exactpro.th2.codec.oracle.logminer.LogMinerTransformer.Companion.truncateFromWhereClause import com.exactpro.th2.codec.oracle.logminer.cfg.LogMinerConfiguration import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.Direction import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.MessageGroup @@ -35,6 +36,8 @@ import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.CsvSource import org.mockito.kotlin.eq import org.mockito.kotlin.mock import org.mockito.kotlin.verify @@ -331,6 +334,18 @@ class LogMinerTransformerTest { } } + @ParameterizedTest + @CsvSource( + """abcWHEREcde,abc;""", + """abc WHERE cde,abc ;""", + """abcwherecde,abc;""", + """abc where cde,abc ;""", + + ) + fun `truncateFromWhereClause test`(source: String, target: String) { + assertEquals(target, truncateFromWhereClause(source)) + } + private fun loadMessages(): List { return LogMinerTransformerTest::class.java.getResourceAsStream( "/com/exactpro/th2/codec/oracle/logminer/log_miner.csv"