update to 0.9.48

bersler · Jul 13, 2022 · 6cce7a9 · 6cce7a9
1 parent 9c12233
commit 6cce7a9
Show file tree

Hide file tree

Showing 22 changed files with 481 additions and 49 deletions.
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,5 +1,5 @@
 0.9.47
-- small fixes
+- fixed old checkpoints deletion
 
 0.9.46
 - small thread fixes

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.16)
-project(OpenLogReplicator VERSION 0.9.47)
+project(OpenLogReplicator VERSION 0.9.48)
 
 set(CMAKE_CXX_STANDARD_REQUIRED TRUE)
 set(CMAKE_CXX_STANDARD 17)

diff --git a/scripts/OpenLogReplicator-example-asm.json b/scripts/OpenLogReplicator-example-asm.json
@@ -29,7 +29,7 @@
   ],
   "target": [
     {
-      "alias": "K1", 
+      "alias": "K1",
       "source": "S1",
       "writer": {
         "type": "kafka",

diff --git a/scripts/gencfg.sql b/scripts/gencfg.sql
@@ -272,7 +272,7 @@ BEGIN
             FROM SYS.OBJ$ AS OF SCN v_SCN O, SYS.CCOL$ AS OF SCN v_SCN L
             WHERE O.OBJ# = L.OBJ# AND O.OWNER# = v_USER_LIST(I) AND
                 (v_USERNAME_LIST(I) <> 'SYS' OR v_SYS_SINGLE = 0
-                OR O.NAME IN ('CCOL$', 'CDEF$', 'COL$', 'DEFERRED_STG$', 'ECOL$', 'OBJ$', 'TAB$', 'TABCOMPART$', 'TABPART$', 'TABSUBPART$', 'USER$'))
+                OR O.NAME IN ('CCOL$', 'CDEF$', 'COL$', 'DEFERRED_STG$', 'ECOL$', 'LOB$', 'OBJ$', 'TAB$', 'TABCOMPART$', 'TABPART$', 'TABSUBPART$', 'USER$'))
             ORDER BY L.ROWID
         ) LOOP
             IF v_PREV = TRUE THEN
@@ -300,7 +300,7 @@ BEGIN
             FROM SYS.OBJ$ AS OF SCN v_SCN O, SYS.CDEF$ AS OF SCN v_SCN D
             WHERE O.OBJ# = D.OBJ# AND O.OWNER# = v_USER_LIST(I) AND
                 (v_USERNAME_LIST(I) <> 'SYS' OR v_SYS_SINGLE = 0
-                OR O.NAME IN ('CCOL$', 'CDEF$', 'COL$', 'DEFERRED_STG$', 'ECOL$', 'OBJ$', 'TAB$', 'TABCOMPART$', 'TABPART$', 'TABSUBPART$', 'USER$'))
+                OR O.NAME IN ('CCOL$', 'CDEF$', 'COL$', 'DEFERRED_STG$', 'ECOL$', 'LOB$', 'OBJ$', 'TAB$', 'TABCOMPART$', 'TABPART$', 'TABSUBPART$', 'USER$'))
             ORDER BY D.ROWID
         ) LOOP
             IF v_PREV = TRUE THEN
@@ -329,7 +329,7 @@ BEGIN
             FROM SYS.OBJ$ AS OF SCN v_SCN O, SYS.COL$ AS OF SCN v_SCN C
             WHERE O.OBJ# = C.OBJ# AND O.OWNER# = v_USER_LIST(I) AND
                 (v_USERNAME_LIST(I) <> 'SYS' OR v_SYS_SINGLE = 0
-                OR O.NAME IN ('CCOL$', 'CDEF$', 'COL$', 'DEFERRED_STG$', 'ECOL$', 'OBJ$', 'TAB$', 'TABCOMPART$', 'TABPART$', 'TABSUBPART$', 'USER$'))
+                OR O.NAME IN ('CCOL$', 'CDEF$', 'COL$', 'DEFERRED_STG$', 'ECOL$', 'LOB$', 'OBJ$', 'TAB$', 'TABCOMPART$', 'TABPART$', 'TABSUBPART$', 'USER$'))
             ORDER BY C.ROWID
         ) LOOP
             IF v_PREV = TRUE THEN
@@ -367,7 +367,7 @@ BEGIN
             FROM SYS.OBJ$ AS OF SCN v_SCN O, SYS.DEFERRED_STG$ AS OF SCN v_SCN DS
             WHERE O.OBJ# = DS.OBJ# AND O.OWNER# = v_USER_LIST(I) AND
                 (v_USERNAME_LIST(I) <> 'SYS' OR v_SYS_SINGLE = 0
-                OR O.NAME IN ('CCOL$', 'CDEF$', 'COL$', 'DEFERRED_STG$', 'ECOL$', 'OBJ$', 'TAB$', 'TABCOMPART$', 'TABPART$', 'TABSUBPART$', 'USER$'))
+                OR O.NAME IN ('CCOL$', 'CDEF$', 'COL$', 'DEFERRED_STG$', 'ECOL$','LOB$',  'OBJ$', 'TAB$', 'TABCOMPART$', 'TABPART$', 'TABSUBPART$', 'USER$'))
             ORDER BY DS.ROWID
         ) LOOP
             IF v_PREV = TRUE THEN
@@ -399,7 +399,7 @@ BEGIN
             FROM SYS.OBJ$ AS OF SCN v_SCN O, SYS.ECOL$ AS OF SCN v_SCN E
             WHERE O.OBJ# = E.TABOBJ# AND O.OWNER# = v_USER_LIST(I) AND
                 (v_USERNAME_LIST(I) <> 'SYS' OR v_SYS_SINGLE = 0
-                OR O.NAME IN ('CCOL$', 'CDEF$', 'COL$', 'DEFERRED_STG$', 'ECOL$', 'OBJ$', 'TAB$', 'TABCOMPART$', 'TABPART$', 'TABSUBPART$', 'USER$'))
+                OR O.NAME IN ('CCOL$', 'CDEF$', 'COL$', 'DEFERRED_STG$', 'ECOL$', 'LOB$', 'OBJ$', 'TAB$', 'TABCOMPART$', 'TABPART$', 'TABSUBPART$', 'USER$'))
             ORDER BY E.ROWID
         ) LOOP
             IF v_PREV = TRUE THEN
@@ -418,6 +418,41 @@ BEGIN
     DBMS_OUTPUT.PUT('],');
     DBMS_OUTPUT.NEW_LINE();
 
+    v_PREV := FALSE;
+    DBMS_OUTPUT.PUT('"sys-lob":[');
+    FOR I IN v_USER_LIST.FIRST .. v_USER_LIST.LAST LOOP
+        FOR v_SYS_LOB IN (
+            SELECT L.ROWID, L.OBJ#, L.COL#, L.INTCOL#, L.LOBJ#
+            FROM SYS.OBJ$ AS OF SCN v_SCN O, SYS.LOB$ AS OF SCN v_SCN L
+            WHERE O.OBJ# = L.OBJ# AND O.OWNER# = v_USER_LIST(I) AND
+                (v_USERNAME_LIST(I) <> 'SYS' OR v_SYS_SINGLE = 0
+                OR O.NAME IN ('CCOL$', 'CDEF$', 'COL$', 'DEFERRED_STG$', 'ECOL$', 'LOB$', 'OBJ$', 'TAB$', 'TABCOMPART$', 'TABPART$', 'TABSUBPART$', 'USER$'))
+            ORDER BY L.ROWID
+        ) LOOP
+            IF v_PREV = TRUE THEN
+                DBMS_OUTPUT.PUT(',');
+            ELSE
+                v_PREV := TRUE;
+            END IF;
+
+            IF v_USERNAME_LIST(I) = 'SYS' THEN
+                v_OBJECT_SINGLE := v_SYS_SINGLE;
+            ELSE
+                v_OBJECT_SINGLE := 0;
+            END IF;
+
+            DBMS_OUTPUT.NEW_LINE();
+            DBMS_OUTPUT.PUT('{"row-id":"' || v_SYS_LOB.ROWID || '"' ||
+                ',"obj":' || v_SYS_LOB.OBJ# ||
+                ',"col":' || v_SYS_LOB.COL# ||
+                ',"int-col":' || v_SYS_LOB.INTCOL# ||
+                ',"obj":' || v_SYS_LOB.OBJ# ||
+                ',"l-obj":' || v_SYS_LOB.LOBJ# || '}');
+        END LOOP;
+    END LOOP;
+    DBMS_OUTPUT.PUT('],');
+    DBMS_OUTPUT.NEW_LINE();
+
     v_PREV := FALSE;
     DBMS_OUTPUT.PUT('"sys-obj":[');
     FOR I IN v_USER_LIST.FIRST .. v_USER_LIST.LAST LOOP
@@ -426,7 +461,7 @@ BEGIN
             MOD(TRUNC(NVL(O.FLAGS, 0) / 18446744073709551616), 18446744073709551616) AS FLAGS2
             FROM SYS.OBJ$ AS OF SCN v_SCN O WHERE O.OWNER# = v_USER_LIST(I) AND
                 (v_USERNAME_LIST(I) <> 'SYS' OR v_SYS_SINGLE = 0
-                OR O.NAME IN ('CCOL$', 'CDEF$', 'COL$', 'DEFERRED_STG$', 'ECOL$', 'OBJ$', 'TAB$', 'TABCOMPART$', 'TABPART$', 'TABSUBPART$', 'USER$'))
+                OR O.NAME IN ('CCOL$', 'CDEF$', 'COL$', 'DEFERRED_STG$', 'ECOL$', 'LOB$', 'OBJ$', 'TAB$', 'TABCOMPART$', 'TABPART$', 'TABSUBPART$', 'USER$'))
             ORDER BY O.ROWID
         ) LOOP
             IF v_PREV = TRUE THEN
@@ -465,7 +500,7 @@ BEGIN
             FROM SYS.OBJ$ AS OF SCN v_SCN O, SYS.TAB$ AS OF SCN v_SCN T
             WHERE O.OBJ# = T.OBJ# AND O.OWNER# = v_USER_LIST(I) AND
                 (v_USERNAME_LIST(I) <> 'SYS' OR v_SYS_SINGLE = 0
-                OR O.NAME IN ('CCOL$', 'CDEF$', 'COL$', 'DEFERRED_STG$', 'ECOL$', 'OBJ$', 'TAB$', 'TABCOMPART$', 'TABPART$', 'TABSUBPART$', 'USER$'))
+                OR O.NAME IN ('CCOL$', 'CDEF$', 'COL$', 'DEFERRED_STG$', 'ECOL$', 'LOB$', 'OBJ$', 'TAB$', 'TABCOMPART$', 'TABPART$', 'TABSUBPART$', 'USER$'))
             ORDER BY T.ROWID
         ) LOOP
             IF v_PREV = TRUE THEN
@@ -500,7 +535,7 @@ BEGIN
             FROM SYS.OBJ$ AS OF SCN v_SCN O, SYS.TABCOMPART$ AS OF SCN v_SCN TCP
             WHERE O.OBJ# = TCP.OBJ# AND O.OWNER# = v_USER_LIST(I) AND
                 (v_USERNAME_LIST(I) <> 'SYS' OR v_SYS_SINGLE = 0
-                OR O.NAME IN ('CCOL$', 'CDEF$', 'COL$', 'DEFERRED_STG$', 'ECOL$', 'OBJ$', 'TAB$', 'TABCOMPART$', 'TABPART$', 'TABSUBPART$', 'USER$'))
+                OR O.NAME IN ('CCOL$', 'CDEF$', 'COL$', 'DEFERRED_STG$', 'ECOL$', 'LOB$', 'OBJ$', 'TAB$', 'TABCOMPART$', 'TABPART$', 'TABSUBPART$', 'USER$'))
             ORDER BY TCP.ROWID
         ) LOOP
             IF v_PREV = TRUE THEN
@@ -533,7 +568,7 @@ BEGIN
             FROM SYS.OBJ$ AS OF SCN v_SCN O, SYS.TABPART$ AS OF SCN v_SCN TP
             WHERE O.OBJ# = TP.OBJ# AND O.OWNER# = v_USER_LIST(I) AND
                 (v_USERNAME_LIST(I) <> 'SYS' OR v_SYS_SINGLE = 0
-                OR O.NAME IN ('CCOL$', 'CDEF$', 'COL$', 'DEFERRED_STG$', 'ECOL$', 'OBJ$', 'TAB$', 'TABCOMPART$', 'TABPART$', 'TABSUBPART$', 'USER$'))
+                OR O.NAME IN ('CCOL$', 'CDEF$', 'COL$', 'DEFERRED_STG$', 'ECOL$', 'LOB$', 'OBJ$', 'TAB$', 'TABCOMPART$', 'TABPART$', 'TABSUBPART$', 'USER$'))
             ORDER BY TP.ROWID
         ) LOOP
             IF v_PREV = TRUE THEN
@@ -566,7 +601,7 @@ BEGIN
             FROM SYS.OBJ$ AS OF SCN v_SCN O, SYS.TABSUBPART$ AS OF SCN v_SCN TSP
             WHERE O.OBJ# = TSP.OBJ# AND O.OWNER# = v_USER_LIST(I) AND
                 (v_USERNAME_LIST(I) <> 'SYS' OR v_SYS_SINGLE = 0
-                OR O.NAME IN ('CCOL$', 'CDEF$', 'COL$', 'DEFERRED_STG$', 'ECOL$', 'OBJ$', 'TAB$', 'TABCOMPART$', 'TABPART$', 'TABSUBPART$', 'USER$'))
+                OR O.NAME IN ('CCOL$', 'CDEF$', 'COL$', 'DEFERRED_STG$', 'ECOL$', 'LOB$', 'OBJ$', 'TAB$', 'TABCOMPART$', 'TABPART$', 'TABSUBPART$', 'USER$'))
             ORDER BY TSP.ROWID
         ) LOOP
             IF v_PREV = TRUE THEN

diff --git a/scripts/grants.sql b/scripts/grants.sql
@@ -22,6 +22,7 @@ GRANT SELECT, FLASHBACK ON SYS.CDEF$ TO <USER>;
 GRANT SELECT, FLASHBACK ON SYS.COL$ TO <USER>;
 GRANT SELECT, FLASHBACK ON SYS.DEFERRED_STG$ TO <USER>;
 GRANT SELECT, FLASHBACK ON SYS.ECOL$ TO <USER>;
+GRANT SELECT, FLASHBACK ON SYS.LOB$ TO <USER>;
 GRANT SELECT, FLASHBACK ON SYS.OBJ$ TO <USER>;
 GRANT SELECT, FLASHBACK ON SYS.TAB$ TO <USER>;
 GRANT SELECT, FLASHBACK ON SYS.TABCOMPART$ TO <USER>;

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -15,6 +15,7 @@ list(APPEND ListCommon
         common/SysCol.cpp
         common/SysDeferredStg.cpp
         common/SysECol.cpp
+        common/SysLob.cpp
         common/SysObj.cpp
         common/SysTab.cpp
         common/SysTabComPart.cpp

diff --git a/src/OpenLogReplicator.cpp b/src/OpenLogReplicator.cpp
@@ -357,6 +357,7 @@ namespace OpenLogReplicator {
             metadata->addElement("SYS", "COL\\$", OPTIONS_SYSTEM_TABLE);
             metadata->addElement("SYS", "DEFERRED_STG\\$", OPTIONS_SYSTEM_TABLE);
             metadata->addElement("SYS", "ECOL\\$", OPTIONS_SYSTEM_TABLE);
+            metadata->addElement("SYS", "LOB\\$", OPTIONS_SYSTEM_TABLE);
             metadata->addElement("SYS", "OBJ\\$", OPTIONS_SYSTEM_TABLE);
             metadata->addElement("SYS", "TAB\\$", OPTIONS_SYSTEM_TABLE);
             metadata->addElement("SYS", "TABPART\\$", OPTIONS_SYSTEM_TABLE);

diff --git a/src/builder/SystemTransaction.cpp b/src/builder/SystemTransaction.cpp
@@ -25,6 +25,7 @@ along with OpenLogReplicator; see the file LICENSE;  If not see
 #include "../common/SysCol.h"
 #include "../common/SysDeferredStg.h"
 #include "../common/SysECol.h"
+#include "../common/SysLob.h"
 #include "../common/SysObj.h"
 #include "../common/SysTab.h"
 #include "../common/SysTabComPart.h"
@@ -48,6 +49,7 @@ namespace OpenLogReplicator {
                 sysCol(nullptr),
                 sysDeferredStg(nullptr),
                 sysECol(nullptr),
+                sysLob(nullptr),
                 sysObj(nullptr),
                 sysTab(nullptr),
                 sysTabComPart(nullptr),
@@ -88,6 +90,11 @@ namespace OpenLogReplicator {
             sysECol = nullptr;
         }
 
+        if (sysLob != nullptr) {
+            delete sysLob;
+            sysLob = nullptr;
+        }
+
         if (sysObj != nullptr) {
             delete sysObj;
             sysObj = nullptr;
@@ -600,6 +607,37 @@ namespace OpenLogReplicator {
             metadata->schema->sysEColTouched = true;
             sysECol = nullptr;
 
+
+        } else if (object->systemTable == TABLE_SYS_LOB) {
+            if (metadata->schema->dictSysLobFind(rowId))
+                throw RuntimeException(std::string("DDL: duplicate SYS.LOB$: (rowid: ") + str + ") for insert");
+            sysLob = new SysLob(rowId, 0, 0, 0, 0, true);
+
+            typeCol column;
+            uint64_t baseMax = builder->valuesMax >> 6;
+            for (uint64_t base = 0; base <= baseMax; ++base) {
+                column = (typeCol)(base << 6);
+                for (uint64_t mask = 1; mask != 0; mask <<= 1, ++column) {
+                    if (builder->valuesSet[base] < mask)
+                        break;
+                    if ((builder->valuesSet[base] & mask) == 0)
+                        continue;
+
+                    if (object->columns[column]->name == "OBJ#")
+                        updateObj(sysLob->obj, column, object, rowId);
+                    else if (object->columns[column]->name == "COL#")
+                        updateNumber16(sysLob->col, 0, column, object, rowId);
+                    else if (object->columns[column]->name == "INTCOL#")
+                        updateNumber16(sysLob->intCol, 0, column, object, rowId);
+                    else if (object->columns[column]->name == "LOBJ#")
+                        updateObj(sysLob->lObj, column, object, rowId);
+                }
+            }
+
+            metadata->schema->sysLobMapRowId[rowId] = sysLob;
+            metadata->schema->sysLobTouched = true;
+            sysLob = nullptr;
+
         } else if (object->systemTable == TABLE_SYS_OBJ) {
             if (metadata->schema->dictSysObjFind(rowId))
                 throw RuntimeException(std::string("DDL: duplicate SYS.OBJ$: (rowid: ") + str + ") for insert");
@@ -1017,6 +1055,50 @@ namespace OpenLogReplicator {
                 }
             }
 
+        } else if (object->systemTable == TABLE_SYS_LOB) {
+            SysLob* sysLob2 = metadata->schema->dictSysLobFind(rowId);
+            if (sysLob2 == nullptr) {
+                TRACE(TRACE2_SYSTEM, "SYSTEM: missing row (rowid: " << rowId << ")")
+                return;
+            }
+
+            typeCol column;
+            uint64_t baseMax = builder->valuesMax >> 6;
+            for (uint64_t base = 0; base <= baseMax; ++base) {
+                column = (typeCol)(base << 6);
+                for (uint64_t mask = 1; mask != 0; mask <<= 1, ++column) {
+                    if (builder->valuesSet[base] < mask)
+                        break;
+                    if ((builder->valuesSet[base] & mask) == 0)
+                        continue;
+
+                    if (object->columns[column]->name == "OBJ#") {
+                        if (updateObj(sysLob2->obj, column, object, rowId)) {
+                            sysLob2->touched = true;
+                            metadata->schema->sysLobTouched = true;
+                        }
+                    } else if (object->columns[column]->name == "COL#") {
+                        if (updateNumber16(sysLob2->col, 0, column, object, rowId)) {
+                            sysLob2->touched = true;
+                            metadata->schema->sysTabTouched = true;
+                            metadata->schema->touchObj(sysLob2->obj);
+                        }
+                    } else if (object->columns[column]->name == "INTCOL#") {
+                        if (updateNumber16(sysLob2->intCol, 0, column, object, rowId)) {
+                            sysLob2->touched = true;
+                            metadata->schema->sysLobTouched = true;
+                            metadata->schema->touchObj(sysLob2->obj);
+                        }
+                    } else if (object->columns[column]->name == "LOBJ#") {
+                        if (updateObj(sysLob2->lObj, column, object, rowId)) {
+                            sysLob2->touched = true;
+                            metadata->schema->sysLobTouched = true;
+                            metadata->schema->touchObj(sysLob2->obj);
+                        }
+                    }
+                }
+            }
+
         } else if (object->systemTable == TABLE_SYS_OBJ) {
             SysObj* sysObj2 = metadata->schema->dictSysObjFind(rowId);
             if (sysObj2 == nullptr) {
@@ -1286,6 +1368,9 @@ namespace OpenLogReplicator {
             case TABLE_SYS_ECOL:
                 metadata->schema->dictSysEColDrop(rowId);
                 break;
+            case TABLE_SYS_LOB:
+                metadata->schema->dictSysLobDrop(rowId);
+                break;
             case TABLE_SYS_OBJ:
                 metadata->schema->dictSysObjDrop(rowId);
                 break;

diff --git a/src/builder/SystemTransaction.h b/src/builder/SystemTransaction.h
@@ -37,6 +37,7 @@ namespace OpenLogReplicator {
     class SysCol;
     class SysDeferredStg;
     class SysECol;
+    class SysLob;
     class SysObj;
     class SysTab;
     class SysTabComPart;
@@ -54,6 +55,7 @@ namespace OpenLogReplicator {
         SysCol* sysCol;
         SysDeferredStg* sysDeferredStg;
         SysECol* sysECol;
+        SysLob* sysLob;
         SysObj* sysObj;
         SysTab* sysTab;
         SysTabComPart* sysTabComPart;

diff --git a/src/common/OracleObject.cpp b/src/common/OracleObject.cpp
@@ -49,6 +49,8 @@ namespace OpenLogReplicator {
                 systemTable = TABLE_SYS_DEFERRED_STG;
             else if (this->name == "ECOL$")
                 systemTable = TABLE_SYS_ECOL;
+            else if (this->name == "LOB$")
+                systemTable = TABLE_SYS_LOB;
             else if (this->name == "OBJ$")
                 systemTable = TABLE_SYS_OBJ;
             else if (this->name == "TAB$")

diff --git a/src/common/SysLob.cpp b/src/common/SysLob.cpp
@@ -0,0 +1,48 @@
+/* Definition of schema SYS.LOB$
+   Copyright (C) 2018-2022 Adam Leszczynski ([email protected])
+
+This file is part of OpenLogReplicator.
+
+OpenLogReplicator is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License as published
+by the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+OpenLogReplicator is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with OpenLogReplicator; see the file LICENSE;  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#include "SysLob.h"
+
+namespace OpenLogReplicator {
+    SysLobKey::SysLobKey(typeObj newObj, typeCol newIntCol) :
+            obj(newObj),
+            intCol(newIntCol) {
+    }
+
+    bool SysLobKey::operator<(const SysLobKey& other) const {
+        if (other.obj > obj)
+            return true;
+        if (other.obj == obj && other.intCol > intCol)
+            return true;
+        return false;
+    }
+
+    SysLob::SysLob(typeRowId& newRowId, typeObj newObj, typeCol newCol, typeCol newIntCol, typeObj newLObj, bool newTouched) :
+            rowId(newRowId),
+            obj(newObj),
+            col(newCol),
+            intCol(newIntCol),
+            lObj(newLObj),
+            touched(newTouched) {
+    }
+
+    bool SysLob::operator!=(const SysLob& other) const {
+        return other.rowId != rowId || other.obj != obj || other.col != col || other.intCol != intCol || other.lObj != lObj;
+    }
+}