Merge pull request #995 from Altinity/2.5.1

2.5.1

.github/workflows/pull-request.yml

@@ -30,6 +30,15 @@ jobs:
     with:
       SINK_CONNECTOR_IMAGE: altinityinfra/clickhouse-sink-connector:${{ github.event.number }}-${{ github.sha }}-lt
 
+  testflows-lightweight-hikari-pool:
+    needs: [build-kafka-lightweight]
+    uses: ./.github/workflows/testflows-sink-connector-lightweight.yml
+    secrets: inherit
+    with:
+      SINK_CONNECTOR_IMAGE: altinityinfra/clickhouse-sink-connector:${{ github.event.number }}-${{ github.sha }}-lt
+      extra_args: --hikari-pool
+      artifact_suffix: hikari-pool
+
   java-tests-kafka:
     needs: [build-kafka-lightweight]
     uses: ./.github/workflows/sink-connector-kafka-tests.yml

.github/workflows/sink-connector-kafka-tests.yml

@@ -19,7 +19,7 @@ env:
 
 jobs:
   java-kafka:
-    runs-on: ubuntu-latest
+    runs-on: [ self-hosted, on-demand, type-cpx51, image-x86-app-docker-ce ]
     steps:
     - uses: actions/checkout@v2
     - name: Set up JDK 17
@@ -28,9 +28,11 @@ jobs:
         java-version: '17'
         distribution: 'temurin'
         cache: maven
+    - name: Install maven
+      run: sudo apt-get update && sudo apt-get install -y maven
     - name: Build with Maven
       working-directory: sink-connector
-      run: mvn -B package --file pom.xml
+      run: mvn -B package -DskipTests=true --file pom.xml
     - name: Run Unit tests
       working-directory: sink-connector
-      run: mvn test -Dgroups=IntegrationTest --file pom.xml
+      run: mvn test --file pom.xml

.github/workflows/testflows-sink-connector-lightweight.yml

@@ -11,11 +11,21 @@ on:
       package:
         description: "Package either 'docker://' or 'https://'. Example: 'https://s3.amazonaws.com/clickhouse-builds/23.3/.../package_release/clickhouse-common-static_23.3.1.64_amd64.deb', or 'docker://altinity/clickhouse-server:23.8.8'"
         type: string
-        default: docker://clickhouse/clickhouse-server:23.3
+        default: docker://clickhouse/clickhouse-server:23.8
       output_format:
         description: "Testflows output style."
         type: string
         default: new-fails
+      extra_args:
+        description: "Extra arguments for the tests."
+        required: false
+        type: string
+        default: ""
+      artifact_suffix:
+        description: "Artifact sufix"
+        required: false
+        type: string
+        default: ""
     secrets:
       DOCKERHUB_USERNAME:
         required: false
@@ -34,11 +44,16 @@ on:
       package:
         description: "Package either 'docker://' or 'https://'. Example: 'https://s3.amazonaws.com/clickhouse-builds/23.3/.../package_release/clickhouse-common-static_23.3.1.64_amd64.deb', or 'docker://altinity/clickhouse-server:23.8.8'"
         type: string
-        default: docker://clickhouse/clickhouse-server:23.3
-      extra_args:
+        default: docker://clickhouse/clickhouse-server:23.8
+      suite:
         description: "Specific Suite To Run (Default * to run everything)."
         required: false
         type: string
+      extra_args:
+        description: "Extra arguments for the tests."
+        required: false
+        type: string
+        default: ""
       custom_run_name:
         description: 'Custom run name (optional)'
         required: false
@@ -115,7 +130,7 @@ jobs:
 
       - name: Run testflows tests
         working-directory: sink-connector-lightweight/tests/integration
-        run: python3 -u regression.py --only "/mysql to clickhouse replication/auto table creation/${{ inputs.extra_args != '' && inputs.extra_args || '*' }}" --clickhouse-binary-path="${{inputs.package}}" --test-to-end --output ${{ inputs.output_format }} --collect-service-logs --attr project="${GITHUB_REPOSITORY}" project.id="$GITHUB_RUN_NUMBER" user.name="$GITHUB_ACTOR" github_actions_run="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" sink_version="registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:latest" s3_url="https://altinity-test-reports.s3.amazonaws.com/index.html#altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_${{github.run.number}}/" --log logs/raw.log
+        run: python3 -u regression.py --only "/mysql to clickhouse replication/auto table creation/${{ inputs.suite != '' && inputs.suite || '*' }}" --clickhouse-binary-path="${{inputs.package}}" --test-to-end ${{ inputs.extra_args != '' && inputs.extra_args || '' }} --output ${{ inputs.output_format }} --collect-service-logs --attr project="${GITHUB_REPOSITORY}" project.id="$GITHUB_RUN_NUMBER" user.name="$GITHUB_ACTOR" github_actions_run="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" sink_version="registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:latest" s3_url="https://altinity-test-reports.s3.amazonaws.com/index.html#altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_${{github.run.number}}/" --log logs/raw.log
 
       - name: Create tfs results report
         if: always()
@@ -139,7 +154,7 @@ jobs:
       - uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: testflows-sink-connector-lightweight-artefacts
+          name: testflows-sink-connector-lightweight-artefacts${{ inputs.artifact_suffix != '' && '-' || '' }}${{ inputs.artifact_suffix }}
           path: |
             sink-connector-lightweight/tests/integration/logs/*.log
             sink-connector-lightweight/tests/integration/env/auto/configs/*.yml
@@ -193,7 +208,7 @@ jobs:
 
       - name: Run testflows tests
         working-directory: sink-connector-lightweight/tests/integration
-        run: python3 -u regression.py --only "/mysql to clickhouse replication/auto replicated table creation/${{ inputs.extra_args != '' && inputs.extra_args || '*' }}" --clickhouse-binary-path="${{inputs.package}}" --test-to-end --output ${{ inputs.output_format }} --collect-service-logs --attr project="${GITHUB_REPOSITORY}" project.id="$GITHUB_RUN_NUMBER" user.name="$GITHUB_ACTOR" github_actions_run="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" sink_version="registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:latest" s3_url="https://altinity-test-reports.s3.amazonaws.com/index.html#altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_${{github.run.number}}/" --log logs/raw.log
+        run: python3 -u regression.py --only "/mysql to clickhouse replication/auto replicated table creation/${{ inputs.suite != '' && inputs.suite || '*' }}" --clickhouse-binary-path="${{inputs.package}}" --test-to-end ${{ inputs.extra_args != '' && inputs.extra_args || '' }} --output ${{ inputs.output_format }} --collect-service-logs --attr project="${GITHUB_REPOSITORY}" project.id="$GITHUB_RUN_NUMBER" user.name="$GITHUB_ACTOR" github_actions_run="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" sink_version="registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:latest" s3_url="https://altinity-test-reports.s3.amazonaws.com/index.html#altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_${{github.run.number}}/" --log logs/raw.log
 
       - name: Create tfs results report
         if: always()
@@ -217,7 +232,7 @@ jobs:
       - uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: testflows-sink-connector-lightweight-replicated-artefacts
+          name: testflows-sink-connector-lightweight-replicated-artefacts${{ inputs.artifact_suffix != '' && '-' || '' }}${{ inputs.artifact_suffix }}
           path: |
             sink-connector-lightweight/tests/integration/logs/*.log
             sink-connector-lightweight/tests/integration/env/auto_replicated/configs/*.yml

release-notes/2.5.1.md

@@ -0,0 +1,19 @@
+## What's Changed
+* Update configuration.md by @subkanthi in https://github.com/Altinity/clickhouse-sink-connector/pull/957
+* Update README.md by @subkanthi in https://github.com/Altinity/clickhouse-sink-connector/pull/961
+* Update getLatestRelease.sh- remove extra quote. by @subkanthi in https://github.com/Altinity/clickhouse-sink-connector/pull/962
+* Update quickstart.md to explain export statement. by @subkanthi in https://github.com/Altinity/clickhouse-sink-connector/pull/963
+* Update architecture.md by @subkanthi in https://github.com/Altinity/clickhouse-sink-connector/pull/964
+* Created replica.status.view for postgres. by @subkanthi in https://github.com/Altinity/clickhouse-sink-connector/pull/970
+* #380 - Added configuration settings to prevent freeze of debezium loop. by @subkanthi in https://github.com/Altinity/clickhouse-sink-connector/pull/974
+* Updated version of upload-artifact to v4. by @subkanthi in https://github.com/Altinity/clickhouse-sink-connector/pull/978
+* Add jdbc params -keepalive by @subkanthi in https://github.com/Altinity/clickhouse-sink-connector/pull/988
+* Update Troubleshooting.md by @subkanthi in https://github.com/Altinity/clickhouse-sink-connector/pull/989
+* Added support to disable connection pool. by @subkanthi in https://github.com/Altinity/clickhouse-sink-connector/pull/992
+* Set nanoseconds in DateTime64 max to zeros and not nines by @subkanthi in https://github.com/Altinity/clickhouse-sink-connector/pull/972
+* Added unit test to validate modify column followed by default NULL by @subkanthi in https://github.com/Altinity/clickhouse-sink-connector/pull/966
+* Fix to limit truncate bigdecimal values. by @subkanthi in https://github.com/Altinity/clickhouse-sink-connector/pull/981
+* Use Hikari Connection Pool for handling Clickhouse connections by @subkanthi in https://github.com/Altinity/clickhouse-sink-connector/pull/893
+
+
+**Full Changelog**: https://github.com/Altinity/clickhouse-sink-connector/compare/2.5.0...2.5.1

sink-connector-lightweight/dependency-reduced-pom.xml

@@ -121,12 +121,6 @@
       <artifactId>javalin-testtools</artifactId>
       <version>5.5.0</version>
       <scope>test</scope>
-      <exclusions>
-        <exclusion>
-          <artifactId>okhttp</artifactId>
-          <groupId>com.squareup.okhttp3</groupId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>io.debezium</groupId>
@@ -162,10 +156,6 @@
           <artifactId>oracle-xe</artifactId>
           <groupId>org.testcontainers</groupId>
         </exclusion>
-        <exclusion>
-          <artifactId>junit-platform-launcher</artifactId>
-          <groupId>org.junit.platform</groupId>
-        </exclusion>
         <exclusion>
           <artifactId>junit-jupiter</artifactId>
           <groupId>org.junit.jupiter</groupId>
@@ -174,10 +164,6 @@
           <artifactId>quarkus-test-common</artifactId>
           <groupId>io.quarkus</groupId>
         </exclusion>
-        <exclusion>
-          <artifactId>okhttp</artifactId>
-          <groupId>com.squareup.okhttp3</groupId>
-        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -207,7 +193,7 @@
     <dependency>
       <groupId>org.testcontainers</groupId>
       <artifactId>testcontainers</artifactId>
-      <version>1.19.1</version>
+      <version>1.20.4</version>
       <scope>test</scope>
       <exclusions>
         <exclusion>
@@ -231,7 +217,7 @@
     <dependency>
       <groupId>org.testcontainers</groupId>
       <artifactId>jdbc</artifactId>
-      <version>1.19.1</version>
+      <version>1.20.4</version>
       <scope>test</scope>
       <exclusions>
         <exclusion>
@@ -243,37 +229,37 @@
     <dependency>
       <groupId>org.testcontainers</groupId>
       <artifactId>junit-jupiter</artifactId>
-      <version>1.19.1</version>
+      <version>1.20.4</version>
       <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.testcontainers</groupId>
       <artifactId>clickhouse</artifactId>
-      <version>1.19.1</version>
+      <version>1.20.4</version>
       <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.testcontainers</groupId>
       <artifactId>mysql</artifactId>
-      <version>1.19.1</version>
+      <version>1.20.4</version>
       <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.testcontainers</groupId>
       <artifactId>postgresql</artifactId>
-      <version>1.19.1</version>
+      <version>1.20.4</version>
       <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.testcontainers</groupId>
       <artifactId>mariadb</artifactId>
-      <version>1.19.1</version>
+      <version>1.20.4</version>
       <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.testcontainers</groupId>
       <artifactId>mongodb</artifactId>
-      <version>1.19.1</version>
+      <version>1.20.4</version>
       <scope>test</scope>
     </dependency>
     <dependency>
@@ -289,10 +275,6 @@
       <version>5.8.1</version>
       <scope>test</scope>
       <exclusions>
-        <exclusion>
-          <artifactId>junit-platform-engine</artifactId>
-          <groupId>org.junit.platform</groupId>
-        </exclusion>
         <exclusion>
           <artifactId>junit-jupiter-api</artifactId>
           <groupId>org.junit.jupiter</groupId>
@@ -317,10 +299,38 @@
           <artifactId>junit-jupiter-api</artifactId>
           <groupId>org.junit.jupiter</groupId>
         </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.platform</groupId>
+      <artifactId>junit-platform-launcher</artifactId>
+      <version>1.8.2</version>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <artifactId>apiguardian-api</artifactId>
+          <groupId>org.apiguardian</groupId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.platform</groupId>
+      <artifactId>junit-platform-engine</artifactId>
+      <version>1.8.2</version>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <artifactId>opentest4j</artifactId>
+          <groupId>org.opentest4j</groupId>
+        </exclusion>
         <exclusion>
-          <artifactId>junit-platform-launcher</artifactId>
+          <artifactId>junit-platform-commons</artifactId>
           <groupId>org.junit.platform</groupId>
         </exclusion>
+        <exclusion>
+          <artifactId>apiguardian-api</artifactId>
+          <groupId>org.apiguardian</groupId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -356,7 +366,7 @@
   <properties>
     <quarkus.platform.version>2.14.0.Final</quarkus.platform.version>
     <quarkus.platform.artifact-id>quarkus-bom</quarkus.platform.artifact-id>
-    <version.testcontainers>1.19.1</version.testcontainers>
+    <version.testcontainers>1.20.4</version.testcontainers>
     <surefire-plugin.version>3.0.0-M7</surefire-plugin.version>
     <version.kafka>3.8.0</version.kafka>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

sink-connector-lightweight/docker/config.yml

@@ -29,7 +29,7 @@ database.server.name: "ER54"
 
 # database.include.list An optional list of regular expressions that match database names to be monitored;
 # any database name not included in the whitelist will be excluded from monitoring. By default all databases will be monitored.
-database.include.list: test
+database.include.list: sbtest
 
 # table.include.list An optional list of regular expressions that match fully-qualified table identifiers for tables to be monitored;
 table.include.list: ""
@@ -130,7 +130,7 @@ database.connectionTimeZone: "UTC"
 
 # Configuration to override the clickhouse database name for a given MySQL database name. If this configuration is not
 # provided, the MySQL database name will be used as the ClickHouse database name.
-clickhouse.database.override.map: "test:ch_test"
+#clickhouse.database.override.map: "test:ch_test"
 
 # clickhouse.datetime.timezone: This timezone will override the default timezone of ClickHouse server. Timezone columns will be set to this timezone.
 #clickhouse.datetime.timezone: "UTC"
@@ -179,6 +179,9 @@ clickhouse.jdbc.params: "keepalive.timeout=3,max_buffer_size=1000000,socket_time
 #Metrics (Prometheus target), required for Grafana Dashboard
 metrics.enable: "true"
 
+#connection.pool.disable: "true"
+#connection.pool.max.size: 500
+
 # Skip schema history capturing, use the following configuration
 # to reduce slow startup when replicating dbs with large number of tables
 #schema.history.internal.store.only.captured.tables.ddl: "true"
@@ -189,4 +192,4 @@ use.nongraceful.disconnect: "true"
 database.keep.alive.interval.`ms: "30000"  #Send keepalive every 30 seconds
 database.connection.reconnect.backoff.ms: "1000"
 database.connection.reconnect.backoff.max.ms: "10000"
-database.ssl.mode: "disabled"
+database.ssl.mode: "disabled"

sink-connector-lightweight/docker/startMySQLCHLocalSinkDev.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+docker network create docker_default
+docker-compose create mysql-master
+docker-compose create clickhouse
+docker-compose start mysql-master
+docker-compose start clickhouse

sink-connector-lightweight/pom.xml

@@ -15,7 +15,7 @@
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <version.debezium>3.0.0.Final</version.debezium>
         <version.junit>5.9.1</version.junit>
-        <version.testcontainers>1.19.1</version.testcontainers>
+        <version.testcontainers>1.20.4</version.testcontainers>
         <version.checkstyle.plugin>3.1.1</version.checkstyle.plugin>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
@@ -233,6 +233,11 @@
             <!-- <scope>test</scope>-->
         </dependency>
 
+        <dependency>
+            <groupId>com.zaxxer</groupId>
+            <artifactId>HikariCP</artifactId>
+            <version>6.0.0</version>
+        </dependency>
         <dependency>
             <groupId>org.antlr</groupId>
             <artifactId>antlr4-runtime</artifactId>
@@ -334,6 +339,18 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>org.junit.platform</groupId>
+            <artifactId>junit-platform-launcher</artifactId>
+            <version>1.8.2</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.platform</groupId>
+            <artifactId>junit-platform-engine</artifactId>
+            <version>1.8.2</version>
+            <scope>test</scope>
+        </dependency>
         <dependency> <!-- necessary for Java 9+ -->
             <groupId>org.apache.tomcat</groupId>
             <artifactId>annotations-api</artifactId>