[NRM-415] Added unescapeQuery

README.md

@@ -1,4 +1,4 @@
-# th2-codec-oracle-log-miner (0.2.1)
+# th2-codec-oracle-log-miner (0.3.0)
 
 ## Description
 
@@ -66,6 +66,7 @@ save-columns: [ OPERATION, SQL_REDO, ROW_ID, TIMESTAMP, TABLE_NAME ]
 **truncate-update-query-from-where-clause** - if true, codec truncates the tail of UPDATE query starting from the WHERE clause before deep parsing.
 This operation improve performance without negative impact, because codec extracts data from the SET clause only.
 **trim-parsed-content** - if true, Codec trims values parsed from `SQL_REDO` field. Default value is `true`
+**unescape-query** - if true, Codec unescapes `SQL_REDO` field by XML standards before parse. Default value is `false`
 **column-prefix** - prefix for parsed columns.
 **save-columns** - set of column names to copy from source message.
 All columns which log miner allow to select are described in the [document](https://docs.oracle.com/en/database/oracle/oracle-database/19/refrn/V-LOGMNR_CONTENTS.html#GUID-B9196942-07BF-4935-B603-FA875064F5C3) 
@@ -79,7 +80,7 @@ metadata:
   name: codec-oracle-log-miner
 spec:
   imageName: ghcr.io/th2-net/th2-codec-oracle-log-miner
-  imageVersion: 0.0.1
+  imageVersion: 0.3.0
   customConfig:
     transportLines:
       rpt:
@@ -92,6 +93,7 @@ spec:
     codecSettings:
       truncate-update-query-from-where-clause: true
       trim-parsed-content: true
+      unescape-query: false
       column-prefix: th2_
       save-columns:
         - OPERATION
@@ -143,6 +145,9 @@ spec:
 
 ## Release notes
 
+### 0.3.0
++ Added `unescape-query` option.
+
 ### 0.2.1
 + Migrated to th2 gradle plugin: `0.0.8`
 + Updated common: `5.12.0-dev`

build.gradle

@@ -44,10 +44,10 @@ dependencies {
     implementation "com.exactpro.th2:codec:5.5.0-dev"
 
     implementation "com.fasterxml.jackson.core:jackson-databind"
+    implementation 'org.apache.commons:commons-text'
 
     implementation "io.github.microutils:kotlin-logging:3.0.5"
 
-    // https://mvnrepository.com/artifact/org.antlr/antlr4
     antlr 'org.antlr:antlr4:4.13.1'
     implementation 'org.antlr:antlr4-runtime:4.13.1'
 
@@ -74,6 +74,10 @@ configurations {
     }
 }
 
+test {
+    useJUnitPlatform()
+}
+
 application {
     mainClass.set("com.exactpro.th2.codec.MainKt")
 }

gradle.properties

@@ -1,3 +1,3 @@
-release_version=0.2.1
+release_version=0.3.0
 description='th2 codec oracle log miner'
 vcs_url=https://github.com/th2-net/th2-codec-oracle-log-miner

src/main/kotlin/com/exactpro/th2/codec/oracle/logminer/LogMinerTransformer.kt

@@ -32,6 +32,7 @@ import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.MessageGro
 import com.exactpro.th2.common.schema.message.impl.rabbitmq.transport.ParsedMessage
 import com.exactpro.th2.common.utils.message.transport.logId
 import mu.KotlinLogging
+import org.apache.commons.text.StringEscapeUtils.unescapeXml
 import kotlin.contracts.ExperimentalContracts
 import kotlin.contracts.contract
 
@@ -56,7 +57,7 @@ class LogMinerTransformer(private val config: LogMinerConfiguration) : IPipeline
                     }
                     val sqlRedo = requireNotNull(message.body[LOG_MINER_SQL_REDO_COLUMN]?.toString()) {
                         "Message doesn't contain required field '$LOG_MINER_SQL_REDO_COLUMN', id: ${message.id.logId}"
-                    }
+                    }.run { if (config.unescapeQuery) unescapeXml(this) else this }
 
                     when (operation) {
                         INSERT.name -> {

src/main/kotlin/com/exactpro/th2/codec/oracle/logminer/cfg/LogMinerConfiguration.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2023 Exactpro (Exactpro Systems Limited)
+ * Copyright 2023-2024 Exactpro (Exactpro Systems Limited)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -37,4 +37,8 @@ class LogMinerConfiguration : IPipelineCodecSettings {
     @JsonProperty("trim-parsed-content")
     @JsonPropertyDescription("Codec trims values parsed from SQL_REDO field")
     var trimParsedContent: Boolean = true
+
+    @JsonProperty("unescape-query")
+    @JsonPropertyDescription("Codec unescapes SQL_REDO field by XML standards before parse")
+    var unescapeQuery: Boolean = false
 }

src/test/kotlin/com/exactpro/th2/codec/oracle/logminer/LogMinerTransformerTest.kt

@@ -18,10 +18,10 @@ package com.exactpro.th2.codec.oracle.logminer
 
 import com.exactpro.th2.codec.api.IReportingContext
 import com.exactpro.th2.codec.oracle.logminer.LogMinerTransformer.Companion.truncateFromWhereClause
-import com.exactpro.th2.codec.oracle.logminer.OPERATIONS.INSERT
-import com.exactpro.th2.codec.oracle.logminer.OPERATIONS.UPDATE
 import com.exactpro.th2.codec.oracle.logminer.OPERATIONS.DELETE
+import com.exactpro.th2.codec.oracle.logminer.OPERATIONS.INSERT
 import com.exactpro.th2.codec.oracle.logminer.OPERATIONS.UNSUPPORTED
+import com.exactpro.th2.codec.oracle.logminer.OPERATIONS.UPDATE
 import com.exactpro.th2.codec.oracle.logminer.antlr.listener.InsertListener
 import com.exactpro.th2.codec.oracle.logminer.antlr.listener.UpdateListener
 import com.exactpro.th2.codec.oracle.logminer.cfg.LogMinerConfiguration
@@ -62,11 +62,22 @@ class LogMinerTransformerTest {
         verifyNoMoreInteractions(reportingContext)
     }
 
-    @Test
-    fun decodesDataUsingDefaultHeader() {
-        val config = LogMinerConfiguration().apply { trimParsedContent = false }
+    @ParameterizedTest
+    @CsvSource(
+        "log_miner.csv, true",
+        "log_miner.csv, false",
+        "log_miner_escaped.csv, true",
+    )
+    fun `decodes data using default header`(
+        fileName: String,
+        unescapeQuery: String,
+    ) {
+        val config = LogMinerConfiguration().apply {
+            this.trimParsedContent = false
+            this.unescapeQuery = unescapeQuery.toBoolean()
+        }
         val codec = LogMinerTransformer(config)
-        val sourceMessages: List<ParsedMessage> = loadMessages()
+        val sourceMessages: List<ParsedMessage> = loadMessages(fileName)
         assertEquals(5, sourceMessages.size)
 
         sourceMessages[0].let { source ->
@@ -421,9 +432,9 @@ class LogMinerTransformerTest {
         assertEquals(target, truncateFromWhereClause(source))
     }
 
-    private fun loadMessages(): List<ParsedMessage> {
+    private fun loadMessages(fileName: String): List<ParsedMessage> {
         return LogMinerTransformerTest::class.java.getResourceAsStream(
-            "/com/exactpro/th2/codec/oracle/logminer/log_miner.csv"
+            "/com/exactpro/th2/codec/oracle/logminer/$fileName"
         ).use { inputStream ->
             requireNotNull(inputStream) {
                 "'log_miner.csv' resource doesn't exist"

src/test/resources/com/exactpro/th2/codec/oracle/logminer/log_miner_escaped.csv

@@ -0,0 +1,20 @@
+SCN,START_SCN,COMMIT_SCN,TIMESTAMP,START_TIMESTAMP,COMMIT_TIMESTAMP,XIDUSN,XIDSLT,XIDSQN,XID,PXIDUSN,PXIDSLT,PXIDSQN,PXID,TX_NAME,OPERATION,OPERATION_CODE,ROLLBACK,SEG_OWNER,SEG_NAME,TABLE_NAME,SEG_TYPE,SEG_TYPE_NAME,TABLE_SPACE,ROW_ID,USERNAME,OS_USERNAME,MACHINE_NAME,AUDIT_SESSIONID,SESSION#,SERIAL#,SESSION_INFO,THREAD#,SEQUENCE#,RBASQN,RBABLK,RBABYTE,UBAFIL,UBABLK,UBAREC,UBASQN,ABS_FILE#,REL_FILE#,DATA_BLK#,DATA_OBJ#,DATA_OBJV#,DATA_OBJD#,SQL_REDO,SQL_UNDO,RS_ID,SSN,CSF,INFO,STATUS,REDO_VALUE,UNDO_VALUE,SAFE_RESUME_SCN,CSCN,OBJECT_ID,EDITION_NAME,CLIENT_ID,SRC_CON_NAME,SRC_CON_ID,SRC_CON_UID,SRC_CON_DBID,SRC_CON_GUID,CON_ID
+3627136,,,06-Dec-2023 08:53:57,,,2,17,652,[B@76c52298,2,17,652,[B@bb9ab64,,DDL,5,0,C##TH2ADMIN,EMPLOYEE,EMPLOYEE,2,TABLE,,AAAAAAAAAAAAAAAAAB,C##TH2ADMIN,oracle,525bc69b1bb7,50030,198,33860,login_username=C##TH2ADMIN client_info= OS_username=oracle Machine_name=525bc69b1bb7 OS_terminal=pts/1 OS_process_id=781 OS_program_name=sqlplus@525bc69b1bb7 (TNS V1-V3),1,1,4,38411,300,4,0,0,0,0,0,0,76665,1,0,"CREATE TABLE EMPLOYEE
+   (
+      ID INT ,
+      NAME CHARACTER (100) ,
+      TITLE CHARACTER (100) ,
+      SALARY CHARACTER (50) ,
+      BONUS_STRUCTURE CHARACTER (50) ,
+      TIME_OFF INT ,
+      SICK_TIME INT ,
+      HEALTH_PLAN CHARACTER (100) ,
+      VISION_PLAN CHARACTER (100) ,
+      DENTAL_PLAN CHARACTER (100) ,
+      PLAN INT ,
+      SAVINGS INT
+   );",, 0x000004.0000960b.012c ,0,0,USER DDL (PlSql=0 RecDep=0),0,510660,510661,,,,ORA$BASE,,CDB$ROOT,1,1,0,,1
+3627247,,,06-Dec-2023 08:54:31,,,9,16,680,[B@a52ca2e,9,16,680,[B@1ad8df52,,INSERT,1,0,C##TH2ADMIN,EMPLOYEE,EMPLOYEE,2,TABLE,USERS,AAASt5AAHAAAAFcAAA,C##TH2ADMIN,user,host,50042,326,23200,login_username=C##TH2ADMIN client_info= OS_username=user Machine_name=host OS_terminal=unknown OS_process_id=1234 OS_program_name=JDBC Thin Client,1,1,4,38478,156,4,16791044,19,133,7,7,348,76665,1,76665,"insert into "C##TH2ADMIN"."EMPLOYEE"("ID","NAME","TITLE","SALARY","BONUS_STRUCTURE","TIME_OFF","SICK_TIME","HEALTH_PLAN","VISION_PLAN","DENTAL_PLAN","PLAN","SAVINGS") values ('1','Chris Montgomery ','Manager ','110,000 ','5% Quarterly ','15','5','Blue Cross and Blue Shield ','Aetna Vision ','Delta Dental ','25000','1');","delete from ""C##TH2ADMIN"".""EMPLOYEE"" where ""ID"" = '1' and ""NAME"" = 'Chris Montgomery ' and ""TITLE"" = 'Manager ' and ""SALARY"" = '110,000 ' and ""BONUS_STRUCTURE"" = '5% Quarterly ' and ""TIME_OFF"" = '15' and ""SICK_TIME"" = '5' and ""HEALTH_PLAN"" = 'Blue Cross and Blue Shield ' and ""VISION_PLAN"" = 'Aetna Vision ' and ""DENTAL_PLAN"" = 'Delta Dental ' and ""PLAN"" = '25000' and ""SAVINGS"" = '1' and ROWID = 'AAASt5AAHAAAAFcAAA';", 0x000004.0000964e.009c ,0,0,,0,510718,510719,,,,,,CDB$ROOT,1,1,0,,1
+3627285,,,06-Dec-2023 08:54:31,,,8,21,689,[B@314c8b4a,8,21,689,[B@26d820eb,,UPDATE,3,0,C##TH2ADMIN,EMPLOYEE,EMPLOYEE,2,TABLE,USERS,AAASt5AAHAAAAFcAAA,C##TH2ADMIN,user,host,50042,326,23200,login_username=C##TH2ADMIN client_info= OS_username=user Machine_name=host OS_terminal=unknown OS_process_id=1234 OS_program_name=JDBC Thin Client,1,1,4,38518,16,4,0,0,0,4,7,348,76665,1,76665,update "C##TH2ADMIN"."EMPLOYEE" set "SAVINGS" = '10' where "SAVINGS" = '1' and ROWID = 'AAASt5AAHAAAAFcAAA';,"update ""C##TH2ADMIN"".""EMPLOYEE"" set ""SAVINGS"" = '1' where ""SAVINGS"" = '10' and ROWID = 'AAASt5AAHAAAAFcAAA';", 0x000004.00009676.0010 ,0,0,,0,510808,510809,,,,,,CDB$ROOT,1,1,0,,1
+3627294,,,06-Dec-2023 08:54:31,,,1,29,644,[B@2c43eb8,1,29,644,[B@7d0cc890,,DELETE,2,0,C##TH2ADMIN,EMPLOYEE,EMPLOYEE,2,TABLE,USERS,AAASt5AAHAAAAFcAAA,C##TH2ADMIN,user,host,50042,326,23200,login_username=C##TH2ADMIN client_info= OS_username=user Machine_name=host OS_terminal=unknown OS_process_id=1234 OS_program_name=JDBC Thin Client,1,1,4,38524,16,4,0,0,0,4,7,348,76665,1,76665,"delete from "C##TH2ADMIN"."EMPLOYEE" where "ID" = '1' and "NAME" = 'Chris Montgomery ' and "TITLE" = 'Manager ' and "SALARY" = '110,000 ' and "BONUS_STRUCTURE" = '5% Quarterly ' and "TIME_OFF" = '15' and "SICK_TIME" = '5' and "HEALTH_PLAN" = 'Blue Cross and Blue Shield ' and "VISION_PLAN" = 'Aetna Vision ' and "DENTAL_PLAN" = 'Delta Dental ' and "PLAN" = '25000' and "SAVINGS" = '10' and ROWID = 'AAASt5AAHAAAAFcAAA';","insert into ""C##TH2ADMIN"".""EMPLOYEE""(""ID"",""NAME"",""TITLE"",""SALARY"",""BONUS_STRUCTURE"",""TIME_OFF"",""SICK_TIME"",""HEALTH_PLAN"",""VISION_PLAN"",""DENTAL_PLAN"",""PLAN"",""SAVINGS"") values ('1','Chris Montgomery ','Manager ','110,000 ','5% Quarterly ','15','5','Blue Cross and Blue Shield ','Aetna Vision ','Delta Dental ','25000','10');", 0x000004.0000967c.0010 ,0,0,,0,510826,510827,,,,,,CDB$ROOT,1,1,0,,1
+2583196,2583196,2583196,06-Dec-2023 08:54:32,06-Dec-2023 08:54:33,06-Dec-2023 08:54:34,10,60,676,0A000600A4020000,10,6,676,0A000600A4020000,,UNSUPPORTED,255,0,SYS,SEG$,SEG$,2,TABLE,SYSTEM,AAAAAIAABAAAB6dAAY,UNKNOWN,UNKNOWN,UNKNOWN,0,0,0,UNKNOWN,1,2,2,40538,16,4,16786509,42,199,1,1,7837,14,0,8,Unsupported,Unsupported,0x000002.00009e5a.0010,0,0,Object or Data type Unsupported,2,1682,1683,0,2583197,,,UNKNOWN,CDB$ROOT,1,1,0,,