Merge branch '3.0' into opti/TD-30681

.github/workflows/taoskeeper-ci.yml

@@ -0,0 +1,58 @@
+name: TaosKeeper CI
+
+on:
+  push:
+    paths:
+      - tools/keeper/**
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    name: Run unit tests
+
+    steps:
+      - name: Checkout the repository
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.18
+
+      - name: Install system dependencies
+        run: |
+          sudo apt update -y
+          sudo apt install -y build-essential cmake libgeos-dev
+
+      - name: Install TDengine
+        run: |
+          mkdir debug
+          cd debug
+          cmake .. -DBUILD_HTTP=false -DBUILD_JDBC=false -DBUILD_TOOLS=false -DBUILD_TEST=off -DBUILD_KEEPER=true
+          make -j 4
+          sudo make install
+          which taosd
+          which taosadapter
+          which taoskeeper
+
+      - name: Start taosd
+        run: |
+          cp /etc/taos/taos.cfg ./
+          sudo echo "supportVnodes 256" >> taos.cfg
+          nohup sudo taosd -c taos.cfg &
+
+      - name: Start taosadapter
+        run: nohup sudo taosadapter &
+
+      - name: Run tests with coverage
+        working-directory: tools/keeper
+        run: |
+          go mod tidy
+          go test -v -coverpkg=./... -coverprofile=coverage.out ./...
+          go tool cover -func=coverage.out
+
+      - name: Clean up
+        if: always()
+        run: |
+          if pgrep taosd; then sudo pkill taosd; fi
+          if pgrep taosadapter; then sudo pkill taosadapter; fi

README-CN.md

@@ -348,7 +348,7 @@ TDengine 提供了丰富的应用程序开发接口，其中包括 C/C++、Java
 
 # 成为社区贡献者
 
-点击 [这里](https://www.taosdata.com/cn/contributor/)，了解如何成为 TDengine 的贡献者。
+点击 [这里](https://www.taosdata.com/contributor)，了解如何成为 TDengine 的贡献者。
 
 # 加入技术交流群
 

cmake/cmake.define

@@ -1,6 +1,7 @@
 cmake_minimum_required(VERSION 3.0)
 set(CMAKE_VERBOSE_MAKEFILE FALSE)
 set(TD_BUILD_TAOSA_INTERNAL FALSE)
+set(TD_BUILD_KEEPER_INTERNAL FALSE)
 
 # set output directory
 SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib)
@@ -57,6 +58,19 @@ IF(TD_BUILD_HTTP)
     ADD_DEFINITIONS(-DHTTP_EMBEDDED)
 ENDIF()
 
+IF("${BUILD_KEEPER}" STREQUAL "")
+    SET(TD_BUILD_KEEPER FALSE)
+ELSEIF(${BUILD_KEEPER} MATCHES "false")
+    SET(TD_BUILD_KEEPER FALSE)
+ELSEIF(${BUILD_KEEPER} MATCHES "true")
+    SET(TD_BUILD_KEEPER TRUE)
+ELSEIF(${BUILD_KEEPER} MATCHES "internal")
+    SET(TD_BUILD_KEEPER FALSE)
+    SET(TD_BUILD_KEEPER_INTERNAL TRUE)
+ELSE()
+    SET(TD_BUILD_KEEPER FALSE)
+ENDIF()
+
 IF("${BUILD_TOOLS}" STREQUAL "")
     IF(TD_LINUX)
         IF(TD_ARM_32)

docs/en/04-get-started/01-docker.md

@@ -75,9 +75,9 @@ taos>
 
 ## TDegnine Graphic User Interface
 
-From TDengine 3.3.0.0, there is a new componenet called `taos-explorer` added in the TDengine docker image. You can use it to manage the databases, super tables, child tables, and data in your TDengine system. There are also some features only available in TDengine Enterprise Edition, please contact TDengine sales team in case you need these features.
+From TDengine 3.3.0.0, there is a new component called `taos-explorer` added in the TDengine docker image. You can use it to manage the databases, super tables, child tables, and data in your TDengine system. There are also some features only available in TDengine Enterprise Edition, please contact TDengine sales team in case you need these features.
 
-To use taos-explorer in the container, you need to access the host port mapped from container port 6060. Assuming the host name is abc.com, and the port used on host is 6060, you need to access `http://abc.com:6060`. taos-explorer uses port 6060 by default in the container. When you use it the first time, you need to register with your enterprise email, then can logon using your user name and password in the TDengine database management system.
+To use taos-explorer in the container, you need to access the host port mapped from container port 6060. Assuming the host name is abc.com, and the port used on host is 6060, you need to access `http://abc.com:6060`. taos-explorer uses port 6060 by default in the container. The default username and password to log in to the TDengine Database Management System is "root/taosdata".
 
 ## Test data insert performance
 

docs/en/14-reference/01-components/10-taosbenchmark

@@ -364,6 +364,9 @@ The configuration parameters for specifying super table tag columns and data col
 - **min**: The minimum value of the column/label of the data type. The generated value will equal or large than the minimum value.
 
 - **max**: The maximum value of the column/label of the data type. The generated value will less than the maximum value.
+
+- **scalingFactor**: Floating-point precision enhancement factor, which takes effect only when the data type is float/double. It has a valid range of positive integers from 1 to 1,000,000. It is used to enhance the precision of generated floating-point numbers, particularly when the min or max values are small. This property enhances the precision after the decimal point by powers of 10: scalingFactor of 10 indicates an enhancement of 1 decimal precision, 100 indicates an enhancement of 2 decimal precision, and so on.
+
 - **fun**: This column of data is filled with functions. Currently, only the sin and cos functions are supported. The input parameter is the timestamp and converted to an angle value. The conversion formula is: angle x=input time column ts value % 360. At the same time, it supports coefficient adjustment and random fluctuation factor adjustment, presented in a fixed format expression, such as fun="10\*sin(x)+100\*random(5)", where x represents the angle, ranging from 0 to 360 degrees, and the growth step size is consistent with the time column step size. 10 represents the coefficient of multiplication, 100 represents the coefficient of addition or subtraction, and 5 represents the fluctuation range within a random range of 5%. The currently supported data types are int, bigint, float, and double. Note: The expression is fixed and cannot be reversed.
 
 - **values**: The value field of the nchar/binary column/label, which will be chosen randomly from the values.

docs/en/14-reference/03-taos-sql/21-node.md

@@ -27,11 +27,15 @@ The preceding SQL command shows all dnodes in the cluster with the ID, endpoint,
 ## Delete a DNODE
 
 ```sql
-DROP DNODE dnode_id
+DROP DNODE dnode_id [force] [unsafe]
 ```
 
 Note that deleting a dnode does not stop its process. You must stop the process after the dnode is deleted.
 
+Only online node is allowed to be deleted. Drop is executed with force option if the offline node need to be deleted.
+
+Drop is executed with unsafe option if the node with single replica is offline, and the data on it is not able to be restored.
+
 ## Modify Dnode Configuration
 
 ```sql

docs/examples/java/src/main/java/com/taos/example/ConsumerLoopFull.java

@@ -1,8 +1,9 @@
 package com.taos.example;
 
-import com.alibaba.fastjson.JSON;
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.taosdata.jdbc.TSDBDriver;
 import com.taosdata.jdbc.tmq.*;
+import com.taosdata.jdbc.utils.JsonUtil;
 
 import java.sql.*;
 import java.time.Duration;
@@ -60,7 +61,7 @@ public static TaosConsumer<ResultBean> getConsumer() throws Exception {
 // ANCHOR_END: create_consumer
     }
 
-    public static void pollExample(TaosConsumer<ResultBean> consumer) throws SQLException {
+    public static void pollExample(TaosConsumer<ResultBean> consumer) throws SQLException, JsonProcessingException {
 // ANCHOR: poll_data_code_piece
         List<String> topics = Collections.singletonList("topic_meters");
         try {
@@ -73,7 +74,7 @@ public static void pollExample(TaosConsumer<ResultBean> consumer) throws SQLExce
                 for (ConsumerRecord<ResultBean> record : records) {
                     ResultBean bean = record.value();
                     // Add your data processing logic here
-                    System.out.println("data: " + JSON.toJSONString(bean));
+                    System.out.println("data: " + JsonUtil.getObjectMapper().writeValueAsString(bean));
                 }
             }
         } catch (Exception ex) {
@@ -91,15 +92,15 @@ public static void pollExample(TaosConsumer<ResultBean> consumer) throws SQLExce
 // ANCHOR_END: poll_data_code_piece
     }
 
-    public static void seekExample(TaosConsumer<ResultBean> consumer) throws SQLException {
+    public static void seekExample(TaosConsumer<ResultBean> consumer) throws SQLException, JsonProcessingException {
 // ANCHOR: consumer_seek
         List<String> topics = Collections.singletonList("topic_meters");
         try {
             // subscribe to the topics
             consumer.subscribe(topics);
             System.out.println("Subscribe topics successfully.");
             Set<TopicPartition> assignment = consumer.assignment();
-            System.out.println("Now assignment: " + JSON.toJSONString(assignment));
+            System.out.println("Now assignment: " + JsonUtil.getObjectMapper().writeValueAsString(assignment));
 
             ConsumerRecords<ResultBean> records = ConsumerRecords.emptyRecord();
             // make sure we have got some data
@@ -125,7 +126,7 @@ public static void seekExample(TaosConsumer<ResultBean> consumer) throws SQLExce
     }
 
 
-    public static void commitExample(TaosConsumer<ResultBean> consumer) throws SQLException {
+    public static void commitExample(TaosConsumer<ResultBean> consumer) throws SQLException, JsonProcessingException {
 // ANCHOR: commit_code_piece
         List<String> topics = Collections.singletonList("topic_meters");
         try {
@@ -135,7 +136,7 @@ public static void commitExample(TaosConsumer<ResultBean> consumer) throws SQLEx
                 for (ConsumerRecord<ResultBean> record : records) {
                     ResultBean bean = record.value();
                     // Add your data processing logic here
-                    System.out.println("data: " + JSON.toJSONString(bean));
+                    System.out.println("data: " + JsonUtil.getObjectMapper().writeValueAsString(bean));
                 }
                 if (!records.isEmpty()) {
                     // after processing the data, commit the offset manually

docs/examples/java/src/main/java/com/taos/example/ConsumerLoopImp.java

@@ -1,7 +1,7 @@
 package com.taos.example;
 
-import com.alibaba.fastjson.JSON;
 import com.taosdata.jdbc.TSDBDriver;
+import com.taosdata.jdbc.utils.JsonUtil;
 
 import java.sql.Connection;
 import java.sql.DriverManager;
@@ -31,7 +31,11 @@ public static void main(String[] args) throws SQLException, InterruptedException
         final AbsConsumerLoop consumerLoop = new AbsConsumerLoop() {
             @Override
             public void process(ResultBean result) {
-                System.out.println("data: " + JSON.toJSONString(result));
+                try{
+                    System.out.println("data: " + JsonUtil.getObjectMapper().writeValueAsString(result));
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
             }
         };
 

docs/examples/java/src/main/java/com/taos/example/WsConsumerLoopFull.java

@@ -1,8 +1,9 @@
 package com.taos.example;
 
-import com.alibaba.fastjson.JSON;
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.taosdata.jdbc.TSDBDriver;
 import com.taosdata.jdbc.tmq.*;
+import com.taosdata.jdbc.utils.JsonUtil;
 
 import java.sql.*;
 import java.time.Duration;
@@ -60,7 +61,7 @@ public static TaosConsumer<ResultBean> getConsumer() throws Exception {
 // ANCHOR_END: create_consumer
     }
 
-    public static void pollExample(TaosConsumer<ResultBean> consumer) throws SQLException {
+    public static void pollExample(TaosConsumer<ResultBean> consumer) throws SQLException, JsonProcessingException {
 // ANCHOR: poll_data_code_piece
         List<String> topics = Collections.singletonList("topic_meters");
         try {
@@ -73,7 +74,7 @@ public static void pollExample(TaosConsumer<ResultBean> consumer) throws SQLExce
                 for (ConsumerRecord<ResultBean> record : records) {
                     ResultBean bean = record.value();
                     // Add your data processing logic here
-                    System.out.println("data: " + JSON.toJSONString(bean));
+                    System.out.println("data: " + JsonUtil.getObjectMapper().writeValueAsString(bean));
                 }
             }
         } catch (Exception ex) {
@@ -91,15 +92,15 @@ public static void pollExample(TaosConsumer<ResultBean> consumer) throws SQLExce
 // ANCHOR_END: poll_data_code_piece
     }
 
-    public static void seekExample(TaosConsumer<ResultBean> consumer) throws SQLException {
+    public static void seekExample(TaosConsumer<ResultBean> consumer) throws SQLException, JsonProcessingException {
 // ANCHOR: consumer_seek
         List<String> topics = Collections.singletonList("topic_meters");
         try {
             // subscribe to the topics
             consumer.subscribe(topics);
             System.out.println("Subscribe topics successfully.");
             Set<TopicPartition> assignment = consumer.assignment();
-            System.out.println("Now assignment: " + JSON.toJSONString(assignment));
+            System.out.println("Now assignment: " + JsonUtil.getObjectMapper().writeValueAsString(assignment));
 
             ConsumerRecords<ResultBean> records = ConsumerRecords.emptyRecord();
             // make sure we have got some data
@@ -125,7 +126,7 @@ public static void seekExample(TaosConsumer<ResultBean> consumer) throws SQLExce
     }
 
 
-    public static void commitExample(TaosConsumer<ResultBean> consumer) throws SQLException {
+    public static void commitExample(TaosConsumer<ResultBean> consumer) throws SQLException, JsonProcessingException {
 // ANCHOR: commit_code_piece
         List<String> topics = Collections.singletonList("topic_meters");
         try {
@@ -135,7 +136,7 @@ public static void commitExample(TaosConsumer<ResultBean> consumer) throws SQLEx
                 for (ConsumerRecord<ResultBean> record : records) {
                     ResultBean bean = record.value();
                     // Add your data processing logic here
-                    System.out.println("data: " + JSON.toJSONString(bean));
+                    System.out.println("data: " + JsonUtil.getObjectMapper().writeValueAsString(bean));
                 }
                 if (!records.isEmpty()) {
                     // after processing the data, commit the offset manually

docs/examples/java/src/main/java/com/taos/example/WsConsumerLoopImp.java

@@ -1,7 +1,7 @@
 package com.taos.example;
 
-import com.alibaba.fastjson.JSON;
 import com.taosdata.jdbc.TSDBDriver;
+import com.taosdata.jdbc.utils.JsonUtil;
 
 import java.sql.Connection;
 import java.sql.DriverManager;
@@ -28,7 +28,11 @@ public static void main(String[] args) throws SQLException, InterruptedException
         final AbsConsumerLoop consumerLoop = new AbsConsumerLoop() {
             @Override
             public void process(ResultBean result) {
-                System.out.println("data: " + JSON.toJSONString(result));
+                try{
+                    System.out.println("data: " + JsonUtil.getObjectMapper().writeValueAsString(result));
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
             }
         };
 

docs/examples/java/src/main/java/com/taos/example/highvolume/DataBaseMonitor.java

@@ -13,6 +13,9 @@ public class DataBaseMonitor {
     public DataBaseMonitor init() throws SQLException {
         if (conn == null) {
             String jdbcURL = System.getenv("TDENGINE_JDBC_URL");
+            if (jdbcURL == null || jdbcURL == ""){
+                jdbcURL = "jdbc:TAOS://localhost:6030?user=root&password=taosdata";
+            }
             conn = DriverManager.getConnection(jdbcURL);
             stmt = conn.createStatement();
         }

docs/examples/java/src/main/java/com/taos/example/highvolume/SQLWriter.java

@@ -69,6 +69,9 @@ public SQLWriter(int maxBatchSize) {
      */
     private static Connection getConnection() throws SQLException {
         String jdbcURL = System.getenv("TDENGINE_JDBC_URL");
+        if (jdbcURL == null || jdbcURL == ""){
+            jdbcURL = "jdbc:TAOS://localhost:6030?user=root&password=taosdata";
+        }
         return DriverManager.getConnection(jdbcURL);
     }
 

docs/examples/java/src/test/java/com/taos/test/TestAll.java

@@ -17,6 +17,37 @@ public void dropDB(String dbName) throws SQLException {
                 stmt.execute("drop database if exists " + dbName);
             }
         }
+        waitTransaction();
+    }
+
+    public void dropTopic(String topicName) throws SQLException {
+        String jdbcUrl = "jdbc:TAOS://localhost:6030?user=root&password=taosdata";
+        try (Connection conn = DriverManager.getConnection(jdbcUrl)) {
+            try (Statement stmt = conn.createStatement()) {
+                stmt.execute("drop topic if exists " + topicName);
+            }
+        }
+        waitTransaction();
+    }
+
+    public void waitTransaction() throws SQLException {
+
+        String jdbcUrl = "jdbc:TAOS://localhost:6030?user=root&password=taosdata";
+        try (Connection conn = DriverManager.getConnection(jdbcUrl)) {
+            try (Statement stmt = conn.createStatement()) {
+                for (int i = 0; i < 10; i++) {
+                    stmt.execute("show transactions");
+                    try (ResultSet resultSet = stmt.getResultSet()) {
+                        if (resultSet.next()) {
+                            int count = resultSet.getInt(1);
+                            if (count == 0) {
+                                break;
+                            }
+                        }
+                    }
+                }
+            }
+        }
     }
 
     public void insertData() throws SQLException {
@@ -104,14 +135,20 @@ public void testConsumer() throws Exception {
         SubscribeDemo.main(args);
     }
 
-//    @Test
-//    public void testSubscribeJni() throws SQLException, InterruptedException {
-//        dropDB("power");
-//        ConsumerLoopFull.main(args);
-//    }
-//    @Test
-//    public void testSubscribeWs() throws SQLException, InterruptedException {
-//        dropDB("power");
-//        WsConsumerLoopFull.main(args);
-//    }
+    @Test
+    public void testSubscribeJni() throws SQLException, InterruptedException {
+        dropTopic("topic_meters");
+        dropDB("power");
+        ConsumerLoopFull.main(args);
+        dropTopic("topic_meters");
+        dropDB("power");
+    }
+    @Test
+    public void testSubscribeWs() throws SQLException, InterruptedException {
+        dropTopic("topic_meters");
+        dropDB("power");
+        WsConsumerLoopFull.main(args);
+        dropTopic("topic_meters");
+        dropDB("power");
+    }
 }