From 6225732fb2b834c2584a3400389ca8bc47b2af9e Mon Sep 17 00:00:00 2001
From: York Cao <52438394+Baymine@users.noreply.github.com>
Date: Mon, 6 Jan 2025 18:38:01 +0800
Subject: [PATCH 001/140] [opt](cache) enhance cache key computation by
 removing comments and trimming SQL input (#46099)

- Currently, the SQL cache system in Doris may miss cache hits due to
semantically identical queries being treated as different because of:
  - Extra whitespace characters in the SQL query
  - SQL comments that don't affect the query execution
- For example, these queries are semantically identical but would
generate different cache keys:
  ```sql
  SELECT * FROM table;
  -- Same query with comments and extra spaces
  /* Comment */  SELECT   *   FROM   table  ;
  ```
- This PR improves the SQL cache hit rate by:
  - Trimming whitespace from SQL queries
  - Removing SQL comments before calculating the cache key MD5
- This ensures that queries that are semantically identical but differ
only in whitespace or comments will now hit the same cache entry,
improving cache efficiency and reducing unnecessary query executions
---
 .../main/java/org/apache/doris/nereids/SqlCacheContext.java    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java
index 29be4af41a76753..20f06e59ce91724 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java
@@ -26,6 +26,7 @@
 import org.apache.doris.mysql.FieldInfo;
 import org.apache.doris.mysql.privilege.DataMaskPolicy;
 import org.apache.doris.mysql.privilege.RowFilterPolicy;
+import org.apache.doris.nereids.parser.NereidsParser;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.Variable;
 import org.apache.doris.nereids.util.Utils;
@@ -349,7 +350,7 @@ public PUniqueId getOrComputeCacheKeyMd5() {
 
     /** doComputeCacheKeyMd5 */
     public synchronized PUniqueId doComputeCacheKeyMd5(Set<Variable> usedVariables) {
-        StringBuilder cacheKey = new StringBuilder(originSql);
+        StringBuilder cacheKey = new StringBuilder(NereidsParser.removeCommentAndTrimBlank(originSql.trim()));
         for (Entry<FullTableName, String> entry : usedViews.entrySet()) {
             cacheKey.append("|")
                     .append(entry.getKey())

From fc8f9f08cd24f067f789bbb582cde72695fed301 Mon Sep 17 00:00:00 2001
From: Dongyang Li <lidongyang@selectdb.com>
Date: Mon, 6 Jan 2025 19:23:45 +0800
Subject: [PATCH 002/140] [chore](ci) support run minio case (#46467)

---
 .../pipeline/vault_p0/conf/regression-conf-custom.groovy | 8 ++++++++
 regression-test/pipeline/vault_p0/run.sh                 | 9 +++++----
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/regression-test/pipeline/vault_p0/conf/regression-conf-custom.groovy b/regression-test/pipeline/vault_p0/conf/regression-conf-custom.groovy
index 3efd22d90c2b706..a55926f736254a7 100644
--- a/regression-test/pipeline/vault_p0/conf/regression-conf-custom.groovy
+++ b/regression-test/pipeline/vault_p0/conf/regression-conf-custom.groovy
@@ -34,3 +34,11 @@ extHiveHmsPort = 7004
 extHdfsPort = 8020
 extHiveServerPort= 7001
 extHiveHmsUser = "root"
+
+// for case test_minio_storage_vault.groovy
+extMinioHost = "127.0.0.1"
+extMinioPort = 19000
+extMinioAk = "minioadmin"
+extMinioSk = "minioadmin"
+extMinioRegion = "us-east-1"
+extMinioBucket = "test-bucket"
diff --git a/regression-test/pipeline/vault_p0/run.sh b/regression-test/pipeline/vault_p0/run.sh
index d0d0e26e733c77b..8322e458a7adc0d 100644
--- a/regression-test/pipeline/vault_p0/run.sh
+++ b/regression-test/pipeline/vault_p0/run.sh
@@ -67,10 +67,11 @@ run() {
     } >>"${teamcity_build_checkoutDir}"/regression-test/pipeline/vault_p0/conf/regression-conf-custom.groovy
     cp -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/vault_p0/conf/regression-conf-custom.groovy \
         "${teamcity_build_checkoutDir}"/regression-test/conf/
-    # # start kafka docker to run case test_rountine_load
-    # sed -i "s/^CONTAINER_UID=\"doris--\"/CONTAINER_UID=\"doris-external--\"/" "${teamcity_build_checkoutDir}"/docker/thirdparties/custom_settings.env
-    # if bash "${teamcity_build_checkoutDir}"/docker/thirdparties/run-thirdparties-docker.sh --stop; then echo; fi
-    # if bash "${teamcity_build_checkoutDir}"/docker/thirdparties/run-thirdparties-docker.sh -c kafka; then echo; else echo "ERROR: start kafka docker failed"; fi
+
+    # start minio docker to run case test_rountine_load
+    sed -i "s/^CONTAINER_UID=\"doris--\"/CONTAINER_UID=\"doris-external--\"/" "${teamcity_build_checkoutDir}"/docker/thirdparties/custom_settings.env
+    if bash "${teamcity_build_checkoutDir}"/docker/thirdparties/run-thirdparties-docker.sh -c minio; then echo; else echo "ERROR: start minio docker failed"; fi
+
     # used to set up HDFS docker
     docker_compose_hdfs_yaml='
 version: "3"

From 6bfb3c8d8b07cf31d6de335d1f630ba65e53254e Mon Sep 17 00:00:00 2001
From: camby <cambyzhu@tencent.com>
Date: Mon, 6 Jan 2025 21:38:55 +0800
Subject: [PATCH 003/140] [fix](mem) heap-buffer-overflow for function
 convert_to (#46405)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What problem does this PR solve?

Reproduce SQL with ASAN version:
`select convert('装装装装装' using gbk);`

Then be crashed:
```
=================================================================
==1830466==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x606002aeec20 at pc 0x560826fb3e66 bp 0x7fc3816a5890 sp 0x7fc3816a5058

WRITE of size 10 at 0x606002aeec20 thread T711 (brpc_light)
```

But if we use release version, we found the result is not correct, and
the memory maybe already corrupted:
```
> select convert('装装装装装' using gbk);
+---------------------------------------------------------+
| convert_to('装装装装装', 'gbk')                         |
+---------------------------------------------------------+
| ~zhuangdang~zhuangdang~zhuangdang~zhuangdang~zhu        |
+---------------------------------------------------------+
1 row in set (0.04 sec)
```

The correct answer should be:
```
> select convert('装装装装装' using gbk);
+--------------------------------------+
| convert_to('装装装装装', 'gbk')      |
+--------------------------------------+
| ~zhuang~zhuang~zhuang~zhuang~zhuang  |
+--------------------------------------+
1 row in set (0.06 sec)
```
---
 be/src/vec/functions/function_string.h        |  13 +++++++++----
 .../nereids_function_p0/scalar_function/C.out |   3 +++
 .../string_functions/test_string_function.out | Bin 4838 -> 4892 bytes
 .../scalar_function/C.groovy                  |   2 +-
 .../test_string_function.groovy               |   2 ++
 5 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/be/src/vec/functions/function_string.h b/be/src/vec/functions/function_string.h
index a729af5948a73fa..1eb437168069b78 100644
--- a/be/src/vec/functions/function_string.h
+++ b/be/src/vec/functions/function_string.h
@@ -3770,9 +3770,10 @@ class FunctionConvertTo : public IFunction {
         auto& res_offset = col_res->get_offsets();
         auto& res_chars = col_res->get_chars();
         res_offset.resize(input_rows_count);
-        // max pinyin size is 6, double of utf8 chinese word 3, add one char to set '~'
-        ColumnString::check_chars_length(str_chars.size() * 2 + input_rows_count, 0);
-        res_chars.resize(str_chars.size() * 2 + input_rows_count);
+        // max pinyin size is 6 + 1 (first '~') for utf8 chinese word 3
+        size_t pinyin_size = (str_chars.size() + 2) / 3 * 7;
+        ColumnString::check_chars_length(pinyin_size, 0);
+        res_chars.resize(pinyin_size);
 
         size_t in_len = 0, out_len = 0;
         for (int i = 0; i < input_rows_count; ++i) {
@@ -3813,7 +3814,11 @@ class FunctionConvertTo : public IFunction {
                     }
 
                     auto end = strchr(buf, ' ');
-                    auto len = end != nullptr ? end - buf : MAX_PINYIN_LEN;
+                    // max len for pinyin is 6
+                    int len = MAX_PINYIN_LEN;
+                    if (end != nullptr && end - buf < MAX_PINYIN_LEN) {
+                        len = end - buf;
+                    }
                     // set first char '~' just make sure all english word lower than chinese word
                     *dest = 126;
                     memcpy(dest + 1, buf, len);
diff --git a/regression-test/data/nereids_function_p0/scalar_function/C.out b/regression-test/data/nereids_function_p0/scalar_function/C.out
index e36fdc7f3742bef..924ddf5b148105e 100644
--- a/regression-test/data/nereids_function_p0/scalar_function/C.out
+++ b/regression-test/data/nereids_function_p0/scalar_function/C.out
@@ -1418,3 +1418,6 @@ Doris	Doris
 -- !convert --
 1	1
 
+-- !convert_1 --
+~zhuang~zhuang~zhuang~zhuang~zhuang
+
diff --git a/regression-test/data/query_p0/sql_functions/string_functions/test_string_function.out b/regression-test/data/query_p0/sql_functions/string_functions/test_string_function.out
index cadf5039794dd87ff4e1af559db1ccb3172a8b23..4af2997eda286b96c77e3edcc4991ab13d04bf13 100644
GIT binary patch
delta 61
qcmaE+I!A580S<%Y{JgT%qLO$+1zlaPx~h!Q#JqHT)aG3rI$Qwt?-*?W

delta 12
TcmbQE_Dprd0glbzIMlcRCHw^^

diff --git a/regression-test/suites/nereids_function_p0/scalar_function/C.groovy b/regression-test/suites/nereids_function_p0/scalar_function/C.groovy
index bf072c9ad8cf74f..0a14602eff9c173 100644
--- a/regression-test/suites/nereids_function_p0/scalar_function/C.groovy
+++ b/regression-test/suites/nereids_function_p0/scalar_function/C.groovy
@@ -197,5 +197,5 @@ suite("nereids_scalar_fn_C") {
 	qt_bitmap_not_nullable "select count(kbitmap) from fn_test_bitmap_not_nullable"
 	qt_char "select char(68, 111, 114, 105, 115), char(68, 111, 114, 105, 115 using utf8);"
 	qt_convert "select convert(1 using gbk), convert(1, string);"
-
+	qt_convert_1 "select convert('装装装装装' using gbk);"
 }
diff --git a/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function.groovy b/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function.groovy
index 6e18fb57eeb4cfe..c1a3e6b57fe8098 100644
--- a/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function.groovy
+++ b/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function.groovy
@@ -163,6 +163,8 @@ suite("test_string_function", "arrow_flight_sql") {
     qt_sql "select right(\"Hello doris\", 120);"
     qt_sql "select right(\"Hello doris\", -6);"
 
+    qt_convert_1 "select convert('装装装装装' using gbk);"
+
     sql """ drop table if exists left_right_test; """
     sql """ create table left_right_test (
         id INT NULL,

From c983e836be861420f665e43fe1acbcfb494ce245 Mon Sep 17 00:00:00 2001
From: James <lijibing@selectdb.com>
Date: Mon, 6 Jan 2025 22:48:40 +0800
Subject: [PATCH 004/140] [test](regression) add log for datetrunc case.
 (#46478)

---
 .../mv/date_trunc/mv_with_date_trunc.groovy   | 201 ++++++++++++++++++
 1 file changed, 201 insertions(+)

diff --git a/regression-test/suites/nereids_rules_p0/mv/date_trunc/mv_with_date_trunc.groovy b/regression-test/suites/nereids_rules_p0/mv/date_trunc/mv_with_date_trunc.groovy
index 115c39f8b503b35..58b2e7bbdd8ca34 100644
--- a/regression-test/suites/nereids_rules_p0/mv/date_trunc/mv_with_date_trunc.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/date_trunc/mv_with_date_trunc.groovy
@@ -98,6 +98,10 @@ suite("mv_with_date_trunc") {
 
     sql """analyze table lineitem with sync;"""
     sql """alter table lineitem modify column l_comment set stats ('row_count'='27');"""
+    def result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
 
     // second, minute ,hour, day, week, month, quarter, year
     // 1. expr in date_trunc is simple col
@@ -127,6 +131,10 @@ suite("mv_with_date_trunc") {
     date_trunc(l_commitdate, 'day');
     """
     order_qt_query1_0_before "${query1_0}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success(db, mv1_0, query1_0, "mv1_0")
     order_qt_query1_0_after "${query1_0}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0"""
@@ -154,6 +162,10 @@ suite("mv_with_date_trunc") {
           l_shipmode;
     """
     order_qt_query1_1_before "${query1_1}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success(db, mv1_1, query1_1, "mv1_1")
     order_qt_query1_1_after "${query1_1}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_1"""
@@ -175,6 +187,10 @@ suite("mv_with_date_trunc") {
           '2023-10-17' <= l_commitdate and '2023-10-18' > l_commitdate;
     """
     order_qt_query1_1_0_before "${query1_1_0}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     // without group, should success
     async_mv_rewrite_success(db, mv1_1_0, query1_1_0, "mv1_1_0")
     order_qt_query1_1_0_after "${query1_1_0}"
@@ -197,6 +213,10 @@ suite("mv_with_date_trunc") {
           '2023-10-17' <= date_trunc(l_commitdate, 'day') and '2023-10-18' > date_trunc(l_commitdate, 'day');
     """
     order_qt_query1_1_1_before "${query1_1_1}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success(db, mv1_1_1, query1_1_1, "mv1_1_1")
     order_qt_query1_1_1_after "${query1_1_1}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_1_1"""
@@ -226,6 +246,10 @@ suite("mv_with_date_trunc") {
     l_shipmode;
     """
     order_qt_query1_2_before "${query1_2}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success(db, mv1_2, query1_2, "mv1_2")
     order_qt_query1_2_after "${query1_2}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_2"""
@@ -254,6 +278,10 @@ suite("mv_with_date_trunc") {
     l_shipmode;
     """
     order_qt_query1_3_before "${query1_3}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success(db, mv1_3, query1_3, "mv1_3")
     order_qt_query1_3_after "${query1_3}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_3"""
@@ -275,6 +303,10 @@ suite("mv_with_date_trunc") {
     '2023-10-16 00:05:08' <= l_shipdate and '2023-10-18' > l_shipdate;
     """
     order_qt_query1_3_0_before "${query1_3_0}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success_without_check_chosen(db, mv1_3_0, query1_3_0, "mv1_3_0")
     order_qt_query1_3_0_after "${query1_3_0}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_3_0"""
@@ -296,6 +328,10 @@ suite("mv_with_date_trunc") {
     '2023-10-16 00:05:08' <= date_trunc(l_shipdate, 'second') and '2023-10-18' > date_trunc(l_shipdate, 'second');
     """
     order_qt_query1_3_1_before "${query1_3_1}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success_without_check_chosen(db, mv1_3_1, query1_3_1, "mv1_3_1")
     order_qt_query1_3_1_after "${query1_3_1}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_3_1"""
@@ -317,6 +353,10 @@ suite("mv_with_date_trunc") {
     '2023-01-17 00:05:08.999999' <= date_trunc(l_commitdate, 'second') and '2023-10-18' > date_trunc(l_commitdate, 'second');
     """
     order_qt_query1_3_2_before "${query1_3_2}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success_without_check_chosen(db, mv1_3_2, query1_3_2, "mv1_3_2")
     order_qt_query1_3_2_after "${query1_3_2}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_3_2"""
@@ -338,6 +378,10 @@ suite("mv_with_date_trunc") {
     '2023-01-17 00:05:08.999999' <= l_commitdate and '2023-10-18' > l_commitdate;
     """
     order_qt_query1_3_3_before "${query1_3_3}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     // use micro second,should fail
     async_mv_rewrite_fail(db, mv1_3_3, query1_3_3, "mv1_3_3")
     order_qt_query1_3_3_after "${query1_3_3}"
@@ -368,6 +412,10 @@ suite("mv_with_date_trunc") {
     l_shipmode;
     """
     order_qt_query1_4_before "${query1_4}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success(db, mv1_4, query1_4, "mv1_4")
     order_qt_query1_4_after "${query1_4}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_4"""
@@ -396,6 +444,10 @@ suite("mv_with_date_trunc") {
     l_shipmode;
     """
     order_qt_query1_5_before "${query1_5}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success(db, mv1_5, query1_5, "mv1_5")
     order_qt_query1_5_after "${query1_5}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_5"""
@@ -417,6 +469,10 @@ suite("mv_with_date_trunc") {
     '2023-10-18' > l_commitdate and '2023-10-16 00:05:00' <= l_commitdate;
     """
     order_qt_query1_5_0_before "${query1_5_0}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success_without_check_chosen(db, mv1_5_0, query1_5_0, "mv1_5_0")
     order_qt_query1_5_0_after "${query1_5_0}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_5_0"""
@@ -437,6 +493,10 @@ suite("mv_with_date_trunc") {
     '2023-10-18' > date_trunc(l_commitdate, 'minute') and '2023-10-16 00:05:00' <= date_trunc(l_commitdate, 'minute');
     """
     order_qt_query1_5_1_before "${query1_5_1}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success_without_check_chosen(db, mv1_5_1, query1_5_1, "mv1_5_1")
     order_qt_query1_5_1_after "${query1_5_1}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_5_1"""
@@ -466,6 +526,10 @@ suite("mv_with_date_trunc") {
     l_shipmode;
     """
     order_qt_query1_6_before "${query1_6}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success(db, mv1_6, query1_6, "mv1_6")
     order_qt_query1_6_after "${query1_6}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_6"""
@@ -494,6 +558,10 @@ suite("mv_with_date_trunc") {
     l_shipmode;
     """
     order_qt_query1_7_before "${query1_7}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success(db, mv1_7, query1_7, "mv1_7")
     order_qt_query1_7_after "${query1_7}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_7"""
@@ -514,6 +582,10 @@ suite("mv_with_date_trunc") {
     '2023-10-16 01:00:00' <= l_commitdate and '2023-10-18' > l_commitdate;
     """
     order_qt_query1_7_0_before "${query1_7_0}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success_without_check_chosen(db, mv1_7_0, query1_7_0, "mv1_7_0")
     order_qt_query1_7_0_after "${query1_7_0}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_7_0"""
@@ -535,6 +607,10 @@ suite("mv_with_date_trunc") {
     '2023-10-16 01:00:00' <= date_trunc(l_commitdate, 'hour') and '2023-10-18' > date_trunc(l_commitdate, 'hour');
     """
     order_qt_query1_7_1_before "${query1_7_1}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success_without_check_chosen(db, mv1_7_1, query1_7_1, "mv1_7_1")
     order_qt_query1_7_1_after "${query1_7_1}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_7_1"""
@@ -564,6 +640,10 @@ suite("mv_with_date_trunc") {
     l_shipmode;
     """
     order_qt_query1_8_before "${query1_8}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success_without_check_chosen(db, mv1_8, query1_8, "mv1_8")
     order_qt_query1_8_after "${query1_8}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_8"""
@@ -592,6 +672,10 @@ suite("mv_with_date_trunc") {
     l_shipmode;
     """
     order_qt_query1_9_before "${query1_9}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success(db, mv1_9, query1_9, "mv1_9")
     order_qt_query1_9_after "${query1_9}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_9"""
@@ -613,6 +697,10 @@ suite("mv_with_date_trunc") {
     '2023-01-16' <= l_commitdate and '2023-09-04' > l_commitdate;
     """
     order_qt_query1_9_0_before "${query1_9_0}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success_without_check_chosen(db, mv1_9_0, query1_9_0, "mv1_9_0")
     order_qt_query1_9_0_after "${query1_9_0}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_9_0"""
@@ -635,6 +723,10 @@ suite("mv_with_date_trunc") {
     '2023-01-16' <= date_trunc(l_commitdate, 'week') and '2023-09-04' > date_trunc(l_commitdate, 'week');
     """
     order_qt_query1_9_1_before "${query1_9_1}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success_without_check_chosen(db, mv1_9_1, query1_9_1, "mv1_9_1")
     order_qt_query1_9_1_after "${query1_9_1}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_9_1"""
@@ -663,6 +755,10 @@ suite("mv_with_date_trunc") {
     l_shipmode;
     """
     order_qt_query1_10_before "${query1_10}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success(db, mv1_10, query1_10, "mv1_10")
     order_qt_query1_10_after "${query1_10}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_10"""
@@ -691,6 +787,10 @@ suite("mv_with_date_trunc") {
     l_shipmode;
     """
     order_qt_query1_11_before "${query1_11}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success(db, mv1_11, query1_11, "mv1_11")
     order_qt_query1_11_after "${query1_11}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_11"""
@@ -712,6 +812,10 @@ suite("mv_with_date_trunc") {
     '2023-02-01' <= l_commitdate and '2023-12-01' > l_commitdate;
     """
     order_qt_query1_11_0_before "${query1_11_0}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success_without_check_chosen(db, mv1_11_0, query1_11_0, "mv1_11_0")
     order_qt_query1_11_0_after "${query1_11_0}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_11_0"""
@@ -733,6 +837,10 @@ suite("mv_with_date_trunc") {
     '2023-02-01' <= date_trunc(l_commitdate, 'month') and '2023-12-01' > date_trunc(l_commitdate, 'month');
     """
     order_qt_query1_11_1_before "${query1_11_1}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success_without_check_chosen(db, mv1_11_1, query1_11_1, "mv1_11_1")
     order_qt_query1_11_1_after "${query1_11_1}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_11_1"""
@@ -762,6 +870,10 @@ suite("mv_with_date_trunc") {
     l_shipmode;
     """
     order_qt_query1_12_before "${query1_12}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success(db, mv1_12, query1_12, "mv1_12")
     order_qt_query1_12_after "${query1_12}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_12"""
@@ -790,6 +902,10 @@ suite("mv_with_date_trunc") {
     l_shipmode;
     """
     order_qt_query1_13_before "${query1_13}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success(db, mv1_13, query1_13, "mv1_13")
     order_qt_query1_13_after "${query1_13}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_13"""
@@ -811,6 +927,10 @@ suite("mv_with_date_trunc") {
     '2023-04-01' <= l_shipdate and '2023-10-01' > l_shipdate;
     """
     order_qt_query1_13_0_before "${query1_13_0}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success_without_check_chosen(db, mv1_13_0, query1_13_0, "mv1_13_0")
     order_qt_query1_13_0_after "${query1_13_0}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_13_0"""
@@ -832,6 +952,10 @@ suite("mv_with_date_trunc") {
     '2023-04-01' <= date_trunc(l_shipdate, 'quarter') and '2023-10-01' > date_trunc(l_shipdate, 'quarter');
     """
     order_qt_query1_13_1_before "${query1_13_1}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success_without_check_chosen(db, mv1_13_1, query1_13_1, "mv1_13_1")
     order_qt_query1_13_1_after "${query1_13_1}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_13_1"""
@@ -861,6 +985,10 @@ suite("mv_with_date_trunc") {
     l_shipmode;
     """
     order_qt_query1_14_before "${query1_14}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success(db, mv1_14, query1_14, "mv1_14")
     order_qt_query1_14_after "${query1_14}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_14"""
@@ -889,6 +1017,10 @@ suite("mv_with_date_trunc") {
     l_shipmode;
     """
     order_qt_query1_15_before "${query1_15}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success(db, mv1_15, query1_15, "mv1_15")
     order_qt_query1_15_after "${query1_15}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_15"""
@@ -910,6 +1042,10 @@ suite("mv_with_date_trunc") {
     '2023-01-01' <= l_commitdate and '2024-01-01' > l_commitdate;
     """
     order_qt_query1_15_0_before "${query1_15_0}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success_without_check_chosen(db, mv1_15_0, query1_15_0, "mv1_15_0")
     order_qt_query1_15_0_after "${query1_15_0}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_15_0"""
@@ -931,6 +1067,10 @@ suite("mv_with_date_trunc") {
     '2023-01-01' <= date_trunc(l_commitdate, 'year') and '2024-01-01' > date_trunc(l_commitdate, 'year');
     """
     order_qt_query1_15_1_before "${query1_15_1}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success_without_check_chosen(db, mv1_15_1, query1_15_1, "mv1_15_1")
     order_qt_query1_15_1_after "${query1_15_1}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_15_1"""
@@ -960,6 +1100,10 @@ suite("mv_with_date_trunc") {
               l_shipmode;
             """
     order_qt_query2_0_before "${query2_0}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_fail(db, mv2_0, query2_0, "mv2_0")
     order_qt_query2_0_after "${query2_0}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_0"""
@@ -988,6 +1132,10 @@ suite("mv_with_date_trunc") {
               l_shipmode;
             """
     order_qt_query2_1_before "${query2_1}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_fail(db, mv2_1, query2_1, "mv2_1")
     order_qt_query2_1_after "${query2_1}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_1"""
@@ -1016,6 +1164,10 @@ suite("mv_with_date_trunc") {
               l_shipmode;
             """
     order_qt_query2_2_before "${query2_2}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     // l_shipdate >= '2023-01-17 00:05:01' because l_shipdate data type is date, so
     // simply to l_shipdate >= '2023-01-18 00:00:00'
     async_mv_rewrite_success(db, mv2_2, query2_2, "mv2_2")
@@ -1046,6 +1198,10 @@ suite("mv_with_date_trunc") {
               l_shipmode;
             """
     order_qt_query2_3_before "${query2_3}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_fail(db, mv2_3, query2_3, "mv2_3")
     order_qt_query2_3_after "${query2_3}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_3"""
@@ -1075,6 +1231,10 @@ suite("mv_with_date_trunc") {
               l_shipmode;
             """
     order_qt_query3_0_before "${query3_0}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_fail(db, mv3_0, query3_0, "mv3_0")
     order_qt_query3_0_after "${query3_0}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_0"""
@@ -1103,6 +1263,10 @@ suite("mv_with_date_trunc") {
               l_shipmode;
             """
     order_qt_query3_1_before "${query3_1}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_fail(db, mv3_1, query3_1, "mv3_1")
     order_qt_query3_1_after "${query3_1}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_1"""
@@ -1131,6 +1295,10 @@ suite("mv_with_date_trunc") {
               l_shipmode;
             """
     order_qt_query3_2_before "${query3_2}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success(db, mv3_2, query3_2, "mv3_2")
     order_qt_query3_2_after "${query3_2}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_2"""
@@ -1159,6 +1327,10 @@ suite("mv_with_date_trunc") {
               l_shipmode;
             """
     order_qt_query3_3_before "${query3_3}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_success(db, mv3_3, query3_3, "mv3_3")
     order_qt_query3_3_after "${query3_3}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_3"""
@@ -1188,6 +1360,10 @@ suite("mv_with_date_trunc") {
               l_shipmode;
             """
     order_qt_query3_4_before "${query3_4}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_fail(db, mv3_4, query3_4, "mv3_4")
     order_qt_query3_4_after "${query3_4}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_4"""
@@ -1217,6 +1393,10 @@ suite("mv_with_date_trunc") {
               l_shipmode;
             """
     order_qt_query3_5_before "${query3_5}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     async_mv_rewrite_fail(db, mv3_5, query3_5, "mv3_5")
     order_qt_query3_5_after "${query3_5}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_5"""
@@ -1263,6 +1443,10 @@ suite("mv_with_date_trunc") {
     l_shipmode,
     date_trunc(l_shipdate, 'day');
     """
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     mv_rewrite_success(query4_0, "mv4_0")
     order_qt_query4_0_after "${query4_0}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv4_0"""
@@ -1276,6 +1460,10 @@ suite("mv_with_date_trunc") {
     lineitem
     where l_shipdate >= '2023-01-01' and l_shipdate < '2023-05-01';
     """
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     mv_rewrite_success_without_check_chosen(query4_1, "mv4_1")
     order_qt_query4_1_after "${query4_1}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv4_1"""
@@ -1309,6 +1497,10 @@ suite("mv_with_date_trunc") {
     date_trunc(l_commitdate_var, 'day');
     """
     order_qt_query5_0_before "${query5_0}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     // should success because as datetime would be datetime(6)
     async_mv_rewrite_success(db, mv5_0, query5_0, "mv5_0")
     order_qt_query5_0_after "${query5_0}"
@@ -1340,6 +1532,10 @@ suite("mv_with_date_trunc") {
     date_trunc(l_commitdate_var, 'day');
     """
     order_qt_query5_1_before "${query5_1}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     // fail because as datetime would be datetime(0), but mv is datetime(6)
     async_mv_rewrite_fail(db, mv5_1, query5_1, "mv5_1")
     order_qt_query5_1_after "${query5_1}"
@@ -1371,8 +1567,13 @@ suite("mv_with_date_trunc") {
     date_trunc(l_commitdate_var, 'minute');
     """
     order_qt_query5_3_before "${query5_3}"
+    result = sql """show table stats lineitem"""
+    logger.info("lineitem table stats: " + result)
+    result = sql """show index stats lineitem lineitem"""
+    logger.info("lineitem index stats: " + result)
     // data is not valid
     async_mv_rewrite_success(db, mv5_3, query5_3, "mv5_3")
     order_qt_query5_3_after "${query5_3}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv5_3"""
 }
+

From a9c3d468b801c19461f641d5bb41b8cb882f400e Mon Sep 17 00:00:00 2001
From: Socrates <suyiteng@selectdb.com>
Date: Tue, 7 Jan 2025 09:44:44 +0800
Subject: [PATCH 005/140] [fix](regression-test) adjust hudi cases (#46473)

### What problem does this PR solve?

This pull request includes changes to multiple test suites for Hudi
tables in the `regression-test/suites/external_table_p2/hudi` directory.
The primary goal of these changes is to ensure that tests are skipped if
the `enableExternalHudiTest` configuration is not set to "true".
Additionally, there is a minor modification to a query in the
`test_hudi_snapshot` suite.

Skipping tests based on configuration:

Added a return statement to skip the test if `enableExternalHudiTest` is
not enabled.

Minor query modification:

*
[`regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy`](diffhunk://#diff-4db2d76dccb6191451d6c0359d0f6636837f6aca5efa6f8425ea5ed4ede9d82aL69-R70):
Changed the order by clause in a query from `signup_date` to `user_id`
to ensure correct ordering.
---
 .../hudi/test_hudi_snapshot.out               | Bin 696105 -> 696105 bytes
 .../hudi/test_hudi_catalog.groovy             |   1 +
 .../hudi/test_hudi_incremental.groovy         |   1 +
 .../hudi/test_hudi_meta.groovy                |   1 +
 .../hudi/test_hudi_orc_tables.groovy          |   1 +
 .../hudi/test_hudi_partition_prune.groovy     |   1 +
 .../hudi/test_hudi_schema_evolution.groovy    |   1 +
 .../hudi/test_hudi_snapshot.groovy            |   3 ++-
 .../hudi/test_hudi_timestamp.groovy           |   1 +
 .../hudi/test_hudi_timetravel.groovy          |   1 +
 10 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/regression-test/data/external_table_p2/hudi/test_hudi_snapshot.out b/regression-test/data/external_table_p2/hudi/test_hudi_snapshot.out
index 1c117c92f6d71214a67c26957654290833914527..114ca5b0a896040cec99a3d30193b758e909f9d2 100644
GIT binary patch
delta 4118
zcmeHKy=qiJ6lU-4W|Mt^s4R9i!4qcY%x|Fv1i?m9_yEq#ok7GDKEZAgxI)0f=2k-5
zfJF*n1TjhL52V;Ysujgn&_{4?b_wJaHn+Pl-{E}ceCO{T9BmvNZA?CNUfvzY7+I7|
z0+Gf7kCA}$R)cmDtr2B@Q)`Sc4nzat!x)3qga|NEB5;KiT&W~P5~(aHM`uZcN1zm_
zZ1BSPr~nZea7zPdWf7vcl=(2$7)x9@pxH-bz@kzfe4q}vX}Cm3Kp-lnv<yPI1wQ)_
z4W3d(6xd7^xHKNL&#X)=gUCi4iJjv!p^E^@$O3nO!Z;Z7%ov>I1OkZ~8Pk!_MZ}GP
z$_~oXNv?q^BZ2Esg5eqOIN>>Ujyh#713-F#7d|J`svv*|!!s8rfjO_y@KOZLE(tl(
zL{Mr?(1P(HM~pOoX`A!vgH}kgK!z_Ih<S!eX0_s=q9(u_E*3q45ImOBD~Xg|5+b}l
z-7Rj^f4}!f)p)a$#+${fr>*QuFME|`Pp7-xPus27OD}fQ<LlMryqhNHy*F)pUze%=
zS@mw#+i!={^x|D{+^Y_;pAK;)J#KscTJ-#Uuh-39PqRORkG-sZ-*oFu)tRQPAA@g2
zb+WgTPWA@JkJ^%0`jW`|zUXzXRKk-7#jR>*Hb^_O)vdM$uE%0nuk$HzcBKxsf31${
z=Rdot84F*F`rq&FG~IhWd|%Y}&sIm(zF159VtD6aJ0_NXOw9iXo7*Af+hMPto=t{d
GZvO{Y4I38#

delta 4118
zcmeHJyJ}QH7-rAzW|MsZF9bWA;2-9GDb#=vu#pr#fHP-i5HW>MaJC2>Az)#1Dj{va
zB84!5n56XrDH2E(QEUZ$1ZNfEE&=0c5`t|1&pCYmclp2d&AIi>xxtIp!)p~4mK$v_
zND>?{$pmO`7=%!Ar)epTRWk$;1{edTedU8M33H&F;=q`epgd(@nMBrm8;p^o^U{&h
zQ!Bu8W`PCG!K$D@m4Z@bDiRFVqYpSH#!GC#Nm&9D8iUfM1kaSw)EMR*9-Y_7C@YW$
z=bZ+IRS5<m2b7yY7!3haGCEJVw4QSZoN);iGYNqafr4YDuq7=WL1yfMQVe^EKsc{}
z8IiyWNgxnOq;^OOLC5ATFJ<Wwl*T6jTu`uvqO@DB1EF4d&kyHqS}6{eP{9xcZW9Fr
zTd)=du*h<fHo*f!hY!-cH_{8D5&~?}XBlUp({F%jiUUQQVm?Yiv$IlMY`(tl&eqj@
zD^~Nxqr25Hv>ah2S0?25OQ^=T@E5u|ceH1*IHWyS=$W+FiT1d&(tfp+?eQ!q3CS^H
zI_mM<5XpqzKYt$2B|B5zLyupm2m9?f*zY`<*!$#DJ<AhkVDF!){#te}HA_$XyYa!Z
zVyjbc&~DtIQ}K4T=W{KY?fFx7z&Z4MmSq0rt~bNG9uL0sUUm}uUEOZxWotJseCWL^
z>h0C3xV_rjTAbYc6R&z%44*+Lq<dd<T4PGMbF;WyFCX;c^1<}NVs<QKA)Hi!KdJkt
z9E3lrK$EJ#-k1)SK26Uy_dmAdFc#hx&9~3(-FW?R|9R2e*qffM*Z53a<Nd3*CNc5f
U$HcGh5L0(p?Z$h9{+lb`0U>P;7XSbN

diff --git a/regression-test/suites/external_table_p2/hudi/test_hudi_catalog.groovy b/regression-test/suites/external_table_p2/hudi/test_hudi_catalog.groovy
index 149eecf5817bd41..ad0f8d25d63665a 100644
--- a/regression-test/suites/external_table_p2/hudi/test_hudi_catalog.groovy
+++ b/regression-test/suites/external_table_p2/hudi/test_hudi_catalog.groovy
@@ -19,6 +19,7 @@ suite("test_hudi_catalog", "p2,external,hudi,external_remote,external_remote_hud
     String enabled = context.config.otherConfigs.get("enableExternalHudiTest")
     if (enabled == null || !enabled.equalsIgnoreCase("true")) {
         logger.info("disable hudi test")
+        return
     }
 
     String catalog_name = "test_hudi_catalog"
diff --git a/regression-test/suites/external_table_p2/hudi/test_hudi_incremental.groovy b/regression-test/suites/external_table_p2/hudi/test_hudi_incremental.groovy
index 885903646cc5b5b..87e3e4a9c6a09a6 100644
--- a/regression-test/suites/external_table_p2/hudi/test_hudi_incremental.groovy
+++ b/regression-test/suites/external_table_p2/hudi/test_hudi_incremental.groovy
@@ -19,6 +19,7 @@ suite("test_hudi_incremental", "p2,external,hudi,external_remote,external_remote
     String enabled = context.config.otherConfigs.get("enableExternalHudiTest")
     if (enabled == null || !enabled.equalsIgnoreCase("true")) {
         logger.info("disable hudi test")
+        return
     }
 
     String catalog_name = "test_hudi_incremental"
diff --git a/regression-test/suites/external_table_p2/hudi/test_hudi_meta.groovy b/regression-test/suites/external_table_p2/hudi/test_hudi_meta.groovy
index 0fe650aded99dc7..efe93d683209978 100644
--- a/regression-test/suites/external_table_p2/hudi/test_hudi_meta.groovy
+++ b/regression-test/suites/external_table_p2/hudi/test_hudi_meta.groovy
@@ -19,6 +19,7 @@ suite("test_hudi_meta", "p2,external,hudi,external_remote,external_remote_hudi")
     String enabled = context.config.otherConfigs.get("enableExternalHudiTest")
     if (enabled == null || !enabled.equalsIgnoreCase("true")) {
         logger.info("disable hudi test")
+        return
     }
 
     String catalog_name = "test_hudi_meta"
diff --git a/regression-test/suites/external_table_p2/hudi/test_hudi_orc_tables.groovy b/regression-test/suites/external_table_p2/hudi/test_hudi_orc_tables.groovy
index 43638a23881e0ef..a57bd61784bb16d 100644
--- a/regression-test/suites/external_table_p2/hudi/test_hudi_orc_tables.groovy
+++ b/regression-test/suites/external_table_p2/hudi/test_hudi_orc_tables.groovy
@@ -19,6 +19,7 @@ suite("test_hudi_orc_tables", "p2,external,hudi,external_remote,external_remote_
     String enabled = context.config.otherConfigs.get("enableExternalHudiTest")
     if (enabled == null || !enabled.equalsIgnoreCase("true")) {
         logger.info("disable hudi test")
+        return
     }
 
     String catalog_name = "test_hudi_orc_tables"
diff --git a/regression-test/suites/external_table_p2/hudi/test_hudi_partition_prune.groovy b/regression-test/suites/external_table_p2/hudi/test_hudi_partition_prune.groovy
index eea17b01711b448..063439d9a87915a 100644
--- a/regression-test/suites/external_table_p2/hudi/test_hudi_partition_prune.groovy
+++ b/regression-test/suites/external_table_p2/hudi/test_hudi_partition_prune.groovy
@@ -19,6 +19,7 @@ suite("test_hudi_partition_prune", "p2,external,hudi,external_remote,external_re
     String enabled = context.config.otherConfigs.get("enableExternalHudiTest")
     if (enabled == null || !enabled.equalsIgnoreCase("true")) {
         logger.info("disable hudi test")
+        return
     }
 
     String catalog_name = "test_hudi_partition_prune"
diff --git a/regression-test/suites/external_table_p2/hudi/test_hudi_schema_evolution.groovy b/regression-test/suites/external_table_p2/hudi/test_hudi_schema_evolution.groovy
index 0da88447cdef154..176484f25c66d24 100644
--- a/regression-test/suites/external_table_p2/hudi/test_hudi_schema_evolution.groovy
+++ b/regression-test/suites/external_table_p2/hudi/test_hudi_schema_evolution.groovy
@@ -19,6 +19,7 @@ suite("test_hudi_schema_evolution", "p2,external,hudi,external_remote,external_r
     String enabled = context.config.otherConfigs.get("enableExternalHudiTest")
     if (enabled == null || !enabled.equalsIgnoreCase("true")) {
         logger.info("disable hudi test")
+        return
     }
 
     String catalog_name = "test_hudi_schema_evolution"
diff --git a/regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy b/regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy
index 0dd7321875b6396..d7f5488c05529d0 100644
--- a/regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy
+++ b/regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy
@@ -19,6 +19,7 @@ suite("test_hudi_snapshot", "p2,external,hudi,external_remote,external_remote_hu
     String enabled = context.config.otherConfigs.get("enableExternalHudiTest")
     if (enabled == null || !enabled.equalsIgnoreCase("true")) {
         logger.info("disable hudi test")
+        return
     }
 
     String catalog_name = "test_hudi_snapshot"
@@ -66,7 +67,7 @@ suite("test_hudi_snapshot", "p2,external,hudi,external_remote,external_remote_hu
         qt_q10 """SELECT * FROM ${table_name} WHERE rating > 4.5 ORDER BY event_time DESC LIMIT 5;"""
 
         // Query all users' signup dates and limit output
-        qt_q11 """SELECT user_id, signup_date FROM ${table_name} ORDER BY signup_date DESC LIMIT 10;"""
+        qt_q11 """SELECT user_id, signup_date FROM ${table_name} ORDER BY user_id DESC LIMIT 10;"""
 
         // Query users with a specific postal code and limit output
         qt_q12 """SELECT * FROM ${table_name} WHERE struct_element(address, 'postal_code') = '80312' ORDER BY event_time LIMIT 5;"""
diff --git a/regression-test/suites/external_table_p2/hudi/test_hudi_timestamp.groovy b/regression-test/suites/external_table_p2/hudi/test_hudi_timestamp.groovy
index 3d7bd40b2d54cff..3c1d01e9f57a76e 100644
--- a/regression-test/suites/external_table_p2/hudi/test_hudi_timestamp.groovy
+++ b/regression-test/suites/external_table_p2/hudi/test_hudi_timestamp.groovy
@@ -19,6 +19,7 @@ suite("test_hudi_timestamp", "p2,external,hudi,external_remote,external_remote_h
     String enabled = context.config.otherConfigs.get("enableExternalHudiTest")
     if (enabled == null || !enabled.equalsIgnoreCase("true")) {
         logger.info("disable hudi test")
+        return
     }
 
     String catalog_name = "test_hudi_timestamp"
diff --git a/regression-test/suites/external_table_p2/hudi/test_hudi_timetravel.groovy b/regression-test/suites/external_table_p2/hudi/test_hudi_timetravel.groovy
index cceeaa412202c60..f1e55a86b83e490 100644
--- a/regression-test/suites/external_table_p2/hudi/test_hudi_timetravel.groovy
+++ b/regression-test/suites/external_table_p2/hudi/test_hudi_timetravel.groovy
@@ -19,6 +19,7 @@ suite("test_hudi_timetravel", "p2,external,hudi,external_remote,external_remote_
     String enabled = context.config.otherConfigs.get("enableExternalHudiTest")
     if (enabled == null || !enabled.equalsIgnoreCase("true")) {
         logger.info("disable hudi test")
+        return
     }
 
     String catalog_name = "test_hudi_timetravel"

From 732d37689ff2b4db4a1f7216ac7773cede6e4bdb Mon Sep 17 00:00:00 2001
From: daidai <changyuwei@selectdb.com>
Date: Tue, 7 Jan 2025 09:53:52 +0800
Subject: [PATCH 006/140] [fix](hms)fix hive catalog follower not get event.
 (#46485)

### What problem does this PR solve?

The property is set in wrong place, so if create catalog on non-master FE,
these 2 properties will only use default value instead of user specified value.
---
 .../datasource/hive/HMSExternalCatalog.java   | 28 +++++++++----------
 .../hive/event/MetastoreEventsProcessor.java  |  2 +-
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
index 2d6c66af444975f..2c80901da4aa45a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
@@ -117,20 +117,6 @@ public void checkProperties() throws DdlException {
             throw new DdlException(
                     "The parameter " + FILE_META_CACHE_TTL_SECOND + " is wrong, value is " + fileMetaCacheTtlSecond);
         }
-        Map<String, String> properties = catalogProperty.getProperties();
-        if (properties.containsKey(HMSProperties.ENABLE_HMS_EVENTS_INCREMENTAL_SYNC)) {
-            enableHmsEventsIncrementalSync =
-                    properties.get(HMSProperties.ENABLE_HMS_EVENTS_INCREMENTAL_SYNC).equals("true");
-        } else {
-            enableHmsEventsIncrementalSync = Config.enable_hms_events_incremental_sync;
-        }
-
-        if (properties.containsKey(HMSProperties.HMS_EVENTIS_BATCH_SIZE_PER_RPC)) {
-            hmsEventsBatchSizePerRpc = Integer.valueOf(properties.get(HMSProperties.HMS_EVENTIS_BATCH_SIZE_PER_RPC));
-        } else {
-            hmsEventsBatchSizePerRpc = Config.hms_events_batch_size_per_rpc;
-        }
-
         // check the dfs.ha properties
         // 'dfs.nameservices'='your-nameservice',
         // 'dfs.ha.namenodes.your-nameservice'='nn1,nn2',
@@ -290,6 +276,20 @@ public void setDefaultPropsIfMissing(boolean isReplay) {
             // always allow fallback to simple auth, so to support both kerberos and simple auth
             catalogProperty.addProperty(DFSFileSystem.PROP_ALLOW_FALLBACK_TO_SIMPLE_AUTH, "true");
         }
+
+        Map<String, String> properties = catalogProperty.getProperties();
+        if (properties.containsKey(HMSProperties.ENABLE_HMS_EVENTS_INCREMENTAL_SYNC)) {
+            enableHmsEventsIncrementalSync =
+                    properties.get(HMSProperties.ENABLE_HMS_EVENTS_INCREMENTAL_SYNC).equals("true");
+        } else {
+            enableHmsEventsIncrementalSync = Config.enable_hms_events_incremental_sync;
+        }
+
+        if (properties.containsKey(HMSProperties.HMS_EVENTIS_BATCH_SIZE_PER_RPC)) {
+            hmsEventsBatchSizePerRpc = Integer.valueOf(properties.get(HMSProperties.HMS_EVENTIS_BATCH_SIZE_PER_RPC));
+        } else {
+            hmsEventsBatchSizePerRpc = Config.hms_events_batch_size_per_rpc;
+        }
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java
index 73054773402bad5..23bf324360bed18 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java
@@ -269,7 +269,7 @@ private NotificationEventResponse getNextEventResponseForSlave(HMSExternalCatalo
 
         // For slave FE nodes, only fetch events which id is lower than masterLastSyncedEventId
         int maxEventSize = Math.min((int) (masterLastSyncedEventId - lastSyncedEventId),
-                Config.hms_events_batch_size_per_rpc);
+                hmsExternalCatalog.getHmsEventsBatchSizePerRpc());
         try {
             return hmsExternalCatalog.getClient().getNextNotification(lastSyncedEventId, maxEventSize, null);
         } catch (MetastoreNotificationFetchException e) {

From aba84f97aa95997e46ba28dacc81b41e837f9f31 Mon Sep 17 00:00:00 2001
From: morrySnow <zhangwenxin@selectdb.com>
Date: Tue, 7 Jan 2025 10:02:33 +0800
Subject: [PATCH 007/140] [fix](Nereids) insert into with cte cannot be
 explained (#46384)

### What problem does this PR solve?

Related PR: #36150

Problem Summary:
not assemble plan with CTE when do explain
---
 .../trees/plans/commands/insert/InsertIntoTableCommand.java | 6 +++++-
 .../suites/nereids_p0/explain/explain_dml.groovy            | 2 ++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java
index 1f1c21d26322b44..c5c13d1a4a1577d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java
@@ -428,7 +428,11 @@ public Plan getExplainPlan(ConnectContext ctx) {
         Optional<CascadesContext> analyzeContext = Optional.of(
                 CascadesContext.initContext(ctx.getStatementContext(), originLogicalQuery, PhysicalProperties.ANY)
         );
-        return InsertUtils.getPlanForExplain(ctx, analyzeContext, getLogicalQuery());
+        Plan plan = InsertUtils.getPlanForExplain(ctx, analyzeContext, getLogicalQuery());
+        if (cte.isPresent()) {
+            plan = cte.get().withChildren(plan);
+        }
+        return plan;
     }
 
     @Override
diff --git a/regression-test/suites/nereids_p0/explain/explain_dml.groovy b/regression-test/suites/nereids_p0/explain/explain_dml.groovy
index a0ff8c3ba7d2f31..8d041e4dd8472cf 100644
--- a/regression-test/suites/nereids_p0/explain/explain_dml.groovy
+++ b/regression-test/suites/nereids_p0/explain/explain_dml.groovy
@@ -119,4 +119,6 @@ suite("explain_dml") {
         sql "update duptbl set v1=1 where k1=1;"
         exception "Only unique table could be updated."
     }
+
+    sql """explain with tmp as (select 1, 2, '3', 4.0, '2000-01-01') insert into epldel1 select * from tmp"""
 }

From ae36af95f50dd2b0dd500c97bcb8193e14af4af4 Mon Sep 17 00:00:00 2001
From: yiguolei <guolei@selectdb.com>
Date: Tue, 7 Jan 2025 10:15:35 +0800
Subject: [PATCH 008/140] [bugfix](be) the config could not be modified on line
 (#46470)

This value is saved in some type description, could not be modified
online. User should restart BE after modify this value.
For example,
    static TypeDescriptor create_string_type() {
        TypeDescriptor ret;
        ret.type = TYPE_STRING;
        ret.len = config::string_type_length_soft_limit_bytes;
        return ret;
    }
---
 be/src/common/config.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 218dc2ae6b152e6..82bfa49e3054488 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -915,7 +915,7 @@ DEFINE_String(rpc_load_balancer, "rr");
 
 // a soft limit of string type length, the hard limit is 2GB - 4, but if too long will cause very low performance,
 // so we set a soft limit, default is 1MB
-DEFINE_mInt32(string_type_length_soft_limit_bytes, "1048576");
+DEFINE_Int32(string_type_length_soft_limit_bytes, "1048576");
 
 DEFINE_Validator(string_type_length_soft_limit_bytes,
                  [](const int config) -> bool { return config > 0 && config <= 2147483643; });

From 4f3b9bb0eb12a666c39c8e4e55d3b3e7703ff2fc Mon Sep 17 00:00:00 2001
From: morrySnow <zhangwenxin@selectdb.com>
Date: Tue, 7 Jan 2025 10:28:35 +0800
Subject: [PATCH 009/140] [fix](variable) variable version not persist in image
 (#46463)

### What problem does this PR solve?

Related PR: #41607 #45648

Problem Summary:
org.apache.doris.qe.GlobalVariable#getPersistentGlobalVarNames should
use bit and to check whether variable is Global
---
 .../src/main/java/org/apache/doris/qe/GlobalVariable.java    | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java
index e3639596ed36bcc..ed4f1757dafb454 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java
@@ -92,7 +92,7 @@ public final class GlobalVariable {
     // 0: table names are stored as specified and comparisons are case sensitive.
     // 1: table names are stored in lowercase on disk and comparisons are not case sensitive.
     // 2: table names are stored as given but compared in lowercase.
-    @VariableMgr.VarAttr(name = LOWER_CASE_TABLE_NAMES, flag = VariableMgr.READ_ONLY)
+    @VariableMgr.VarAttr(name = LOWER_CASE_TABLE_NAMES, flag = VariableMgr.READ_ONLY | VariableMgr.GLOBAL)
     public static int lowerCaseTableNames = 0;
 
     @VariableMgr.VarAttr(name = LICENSE, flag = VariableMgr.READ_ONLY)
@@ -200,8 +200,7 @@ public static List<String> getPersistentGlobalVarNames() {
         List<String> varNames = Lists.newArrayList();
         for (Field field : GlobalVariable.class.getDeclaredFields()) {
             VariableMgr.VarAttr attr = field.getAnnotation(VariableMgr.VarAttr.class);
-            // Since the flag of lower_case_table_names is READ_ONLY, it is handled separately here.
-            if (attr != null && (attr.flag() == VariableMgr.GLOBAL || attr.name().equals(LOWER_CASE_TABLE_NAMES))) {
+            if (attr != null && (attr.flag() & VariableMgr.GLOBAL) != 0) {
                 varNames.add(attr.name());
             }
         }

From e718c0b8d1ae104b5d7c005b6fe75246a6296e1e Mon Sep 17 00:00:00 2001
From: 924060929 <lanhuajian@selectdb.com>
Date: Tue, 7 Jan 2025 10:44:54 +0800
Subject: [PATCH 010/140] [fix](nereids) fix sql cache bug and some tests
 (#46443)

1. use retry to fix unstable test `colocate_union_numbers`,
`prune_bucket_with_bucket_shuffle_join`
2. fix failed test `explain`, this bug only exists in master branch,
introduced by #40202
3. fix sql cache bug which use stale cache after drop table and
create(table id changed), test in `parse_sql_from_sql_cache`, introduced
by #33262
4. regression test add `foreachFrontends`, `foreachBackends`, `retry`
function
---
 .../common/cache/NereidsSqlCacheManager.java  |   62 +-
 .../apache/doris/nereids/SqlCacheContext.java |   28 +-
 .../doris/nereids/StatementContext.java       |    2 +-
 .../nereids/trees/plans/AbstractPlan.java     |    4 +-
 .../doris/regression/suite/Suite.groovy       |   39 +-
 .../cache/parse_sql_from_sql_cache.groovy     | 1536 ++++++++---------
 .../distribute/colocate_union_numbers.groovy  |   28 +-
 ...une_bucket_with_bucket_shuffle_join.groovy |   28 +-
 .../suites/nereids_syntax_p0/explain.groovy   |    5 +-
 9 files changed, 898 insertions(+), 834 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java
index aba0decb76e3a01..2aca4d5f1efac75 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java
@@ -23,6 +23,7 @@
 import org.apache.doris.catalog.OlapTable;
 import org.apache.doris.catalog.Partition;
 import org.apache.doris.catalog.TableIf;
+import org.apache.doris.catalog.TableIf.TableType;
 import org.apache.doris.catalog.View;
 import org.apache.doris.common.Config;
 import org.apache.doris.common.ConfigBase.DefaultConfHandler;
@@ -39,6 +40,7 @@
 import org.apache.doris.nereids.SqlCacheContext.FullColumnName;
 import org.apache.doris.nereids.SqlCacheContext.FullTableName;
 import org.apache.doris.nereids.SqlCacheContext.ScanTable;
+import org.apache.doris.nereids.SqlCacheContext.TableVersion;
 import org.apache.doris.nereids.StatementContext;
 import org.apache.doris.nereids.analyzer.UnboundVariable;
 import org.apache.doris.nereids.parser.NereidsParser;
@@ -199,14 +201,14 @@ public Optional<LogicalSqlCache> tryParseSql(ConnectContext connectContext, Stri
                     .getSqlCacheContext().ifPresent(ctx -> ctx.setCacheKeyType(CacheKeyType.MD5));
 
             if (sqlCacheContextWithVariable != null) {
-                return tryParseSqlWithoutCheckVariable(
-                        connectContext, md5CacheKey, sqlCacheContextWithVariable, currentUserIdentity
+                return tryParseSql(
+                        connectContext, md5CacheKey, sqlCacheContextWithVariable, currentUserIdentity, true
                 );
             } else {
                 return Optional.empty();
             }
         } else {
-            return tryParseSqlWithoutCheckVariable(connectContext, key, sqlCacheContext, currentUserIdentity);
+            return tryParseSql(connectContext, key, sqlCacheContext, currentUserIdentity, false);
         }
     }
 
@@ -223,9 +225,9 @@ private String normalizeSql(String sql) {
         return NereidsParser.removeCommentAndTrimBlank(sql);
     }
 
-    private Optional<LogicalSqlCache> tryParseSqlWithoutCheckVariable(
-            ConnectContext connectContext, String key,
-            SqlCacheContext sqlCacheContext, UserIdentity currentUserIdentity) {
+    private Optional<LogicalSqlCache> tryParseSql(
+            ConnectContext connectContext, String key, SqlCacheContext sqlCacheContext,
+            UserIdentity currentUserIdentity, boolean checkUserVariable) {
         Env env = connectContext.getEnv();
 
         if (!tryLockTables(connectContext, env, sqlCacheContext)) {
@@ -259,8 +261,12 @@ private Optional<LogicalSqlCache> tryParseSqlWithoutCheckVariable(
         try {
             Optional<ResultSet> resultSetInFe = sqlCacheContext.getResultSetInFe();
 
-            List<Variable> currentVariables = resolveUserVariables(sqlCacheContext);
-            boolean usedVariablesChanged = usedVariablesChanged(currentVariables, sqlCacheContext);
+            List<Variable> currentVariables = ImmutableList.of();
+            if (checkUserVariable) {
+                currentVariables = resolveUserVariables(sqlCacheContext);
+            }
+            boolean usedVariablesChanged
+                    = checkUserVariable && usedVariablesChanged(currentVariables, sqlCacheContext);
             if (resultSetInFe.isPresent() && !usedVariablesChanged) {
                 MetricRepo.COUNTER_CACHE_HIT_SQL.increase(1L);
 
@@ -274,9 +280,15 @@ private Optional<LogicalSqlCache> tryParseSqlWithoutCheckVariable(
             }
 
             Status status = new Status();
-            PUniqueId cacheKeyMd5 = usedVariablesChanged
-                    ? sqlCacheContext.doComputeCacheKeyMd5(Utils.fastToImmutableSet(currentVariables))
-                    : sqlCacheContext.getOrComputeCacheKeyMd5();
+
+            PUniqueId cacheKeyMd5;
+            if (usedVariablesChanged) {
+                invalidateCache(key);
+                cacheKeyMd5 = sqlCacheContext.doComputeCacheKeyMd5(Utils.fastToImmutableSet(currentVariables));
+            } else {
+                cacheKeyMd5 = sqlCacheContext.getOrComputeCacheKeyMd5();
+            }
+
             InternalService.PFetchCacheResult cacheData =
                     SqlCache.getCacheData(sqlCacheContext.getCacheProxy(),
                             cacheKeyMd5, sqlCacheContext.getLatestPartitionId(),
@@ -308,20 +320,36 @@ private boolean tablesOrDataChanged(Env env, SqlCacheContext sqlCacheContext) {
             return true;
         }
 
-        for (ScanTable scanTable : sqlCacheContext.getScanTables()) {
-            FullTableName fullTableName = scanTable.fullTableName;
-            TableIf tableIf = findTableIf(env, fullTableName);
-            if (!(tableIf instanceof OlapTable)) {
+        // the query maybe scan empty partition of the table, we should check these table version too,
+        // but the table not exists in sqlCacheContext.getScanTables(), so we need check here.
+        // check table type and version
+        for (Entry<FullTableName, TableVersion> scanTable : sqlCacheContext.getUsedTables().entrySet()) {
+            TableVersion tableVersion = scanTable.getValue();
+            if (tableVersion.type != TableType.OLAP) {
+                return true;
+            }
+            TableIf tableIf = findTableIf(env, scanTable.getKey());
+            if (!(tableIf instanceof OlapTable) || tableVersion.id != tableIf.getId()) {
                 return true;
             }
+
             OlapTable olapTable = (OlapTable) tableIf;
             long currentTableVersion = olapTable.getVisibleVersion();
-            long cacheTableVersion = scanTable.latestVersion;
+            long cacheTableVersion = tableVersion.version;
             // some partitions have been dropped, or delete or updated or replaced, or insert rows into new partition?
             if (currentTableVersion != cacheTableVersion) {
                 return true;
             }
+        }
 
+        // check partition version
+        for (ScanTable scanTable : sqlCacheContext.getScanTables()) {
+            FullTableName fullTableName = scanTable.fullTableName;
+            TableIf tableIf = findTableIf(env, fullTableName);
+            if (!(tableIf instanceof OlapTable)) {
+                return true;
+            }
+            OlapTable olapTable = (OlapTable) tableIf;
             Collection<Long> partitionIds = scanTable.getScanPartitions();
             olapTable.getVersionInBatchForCloudMode(partitionIds);
 
@@ -392,7 +420,7 @@ private boolean dataMaskPoliciesChanged(
      */
     private boolean tryLockTables(ConnectContext connectContext, Env env, SqlCacheContext sqlCacheContext) {
         StatementContext currentStatementContext = connectContext.getStatementContext();
-        for (FullTableName fullTableName : sqlCacheContext.getUsedTables()) {
+        for (FullTableName fullTableName : sqlCacheContext.getUsedTables().keySet()) {
             TableIf tableIf = findTableIf(env, fullTableName);
             if (tableIf == null) {
                 return false;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java
index 20f06e59ce91724..2278436888b2f67 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java
@@ -20,7 +20,9 @@
 import org.apache.doris.analysis.Expr;
 import org.apache.doris.analysis.UserIdentity;
 import org.apache.doris.catalog.DatabaseIf;
+import org.apache.doris.catalog.OlapTable;
 import org.apache.doris.catalog.TableIf;
+import org.apache.doris.catalog.TableIf.TableType;
 import org.apache.doris.common.Pair;
 import org.apache.doris.datasource.CatalogIf;
 import org.apache.doris.mysql.FieldInfo;
@@ -42,6 +44,7 @@
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 
+import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -62,7 +65,8 @@ public class SqlCacheContext {
     private volatile long latestPartitionTime = -1;
     private volatile long latestPartitionVersion = -1;
     private volatile long sumOfPartitionNum = -1;
-    private final Set<FullTableName> usedTables = Sets.newLinkedHashSet();
+    // value: version of table
+    private final Map<FullTableName, TableVersion> usedTables = Maps.newLinkedHashMap();
     // value: ddl sql
     private final Map<FullTableName, String> usedViews = Maps.newLinkedHashMap();
     // value: usedColumns
@@ -136,8 +140,13 @@ public synchronized void addUsedTable(TableIf tableIf) {
             return;
         }
 
-        usedTables.add(
-                new FullTableName(database.getCatalog().getName(), database.getFullName(), tableIf.getName())
+        usedTables.put(
+                new FullTableName(database.getCatalog().getName(), database.getFullName(), tableIf.getName()),
+                new TableVersion(
+                        tableIf.getId(),
+                        tableIf instanceof OlapTable ? ((OlapTable) tableIf).getVisibleVersion() : 0L,
+                        tableIf.getType()
+                )
         );
     }
 
@@ -283,8 +292,8 @@ public void setCacheProxy(CacheProxy cacheProxy) {
         this.cacheProxy = cacheProxy;
     }
 
-    public Set<FullTableName> getUsedTables() {
-        return ImmutableSet.copyOf(usedTables);
+    public Map<FullTableName, TableVersion> getUsedTables() {
+        return Collections.unmodifiableMap(usedTables);
     }
 
     public Map<FullTableName, String> getUsedViews() {
@@ -459,6 +468,15 @@ public void addScanPartition(Long partitionId) {
         }
     }
 
+    /** TableVersion */
+    @lombok.Data
+    @lombok.AllArgsConstructor
+    public static class TableVersion {
+        public final long id;
+        public final long version;
+        public final TableType type;
+    }
+
     /** CacheKeyType */
     public enum CacheKeyType {
         // use `userIdentity`:`sql`.trim() as Cache key in FE
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
index 0b671ebdb71204a..597cef2d47e8c15 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
@@ -235,7 +235,7 @@ public StatementContext(ConnectContext connectContext, OriginStatement originSta
             this.sqlCacheContext = new SqlCacheContext(
                     connectContext.getCurrentUserIdentity(), connectContext.queryId());
             if (originStatement != null) {
-                this.sqlCacheContext.setOriginSql(originStatement.originStmt.trim());
+                this.sqlCacheContext.setOriginSql(originStatement.originStmt);
             }
         } else {
             this.sqlCacheContext = null;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java
index eb65048050fda10..958b4fe9c424b4a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java
@@ -228,6 +228,8 @@ public List<Plan> getAncestors() {
     }
 
     public void updateActualRowCount(long actualRowCount) {
-        statistics.setActualRowCount(actualRowCount);
+        if (statistics != null) {
+            statistics.setActualRowCount(actualRowCount);
+        }
     }
 }
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index 64c99d1a114d556..7f21f45774bd6ea 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -25,6 +25,7 @@ import com.google.common.collect.Maps
 import com.google.common.util.concurrent.Futures
 import com.google.common.util.concurrent.ListenableFuture
 import com.google.common.util.concurrent.MoreExecutors
+import com.google.common.util.concurrent.Uninterruptibles
 import com.google.gson.Gson
 import groovy.json.JsonOutput
 import groovy.json.JsonSlurper
@@ -37,7 +38,6 @@ import org.apache.doris.regression.RegressionTest
 import org.apache.doris.regression.action.BenchmarkAction
 import org.apache.doris.regression.action.ProfileAction
 import org.apache.doris.regression.action.WaitForAction
-import org.apache.doris.regression.util.DataUtils
 import org.apache.doris.regression.util.OutputUtils
 import org.apache.doris.regression.action.CreateMVAction
 import org.apache.doris.regression.action.ExplainAction
@@ -59,13 +59,7 @@ import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 
 import java.sql.Connection
-import java.io.File
-import java.math.BigDecimal;
-import java.sql.PreparedStatement
-import java.sql.ResultSetMetaData
-import java.util.Map;
 import java.util.concurrent.Callable
-import java.util.concurrent.ExecutorService
 import java.util.concurrent.Executors
 import java.util.concurrent.Future
 import java.util.concurrent.ThreadFactory
@@ -701,6 +695,23 @@ class Suite implements GroovyInterceptable {
         return sql
     }
 
+    <T> T retry(int executeTimes = 3, int intervalMillis = 1000, Closure<Integer> closure) {
+        Throwable throwable = null
+        for (int i = 1; i <= executeTimes; ++i) {
+            try {
+                return closure(i) as T
+            } catch (Throwable t) {
+                logger.warn("Retry failed: $t", t)
+                throwable = t
+                Uninterruptibles.sleepUninterruptibly(intervalMillis, TimeUnit.MILLISECONDS)
+            }
+        }
+        if (throwable != null) {
+            throw throwable
+        }
+        return null
+    }
+
     void explain(Closure actionSupplier) {
         if (context.useArrowFlightSql()) {
             runAction(new ExplainAction(context, "ARROW_FLIGHT_SQL"), actionSupplier)
@@ -1073,6 +1084,20 @@ class Suite implements GroovyInterceptable {
         }
     }
 
+    void foreachFrontends(Closure action) {
+        def rows = sql_return_maparray("show frontends")
+        for (def row in rows) {
+            action(row)
+        }
+    }
+
+    void foreachBackends(Closure action) {
+        def rows = sql_return_maparray("show backends")
+        for (def row in rows) {
+            action(row)
+        }
+    }
+
     List<String> getFrontendIpHttpPort() {
         return sql_return_maparray("show frontends").collect { it.Host + ":" + it.HttpPort };
     }
diff --git a/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy b/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy
index a9f833d7021e367..e7fb5f3da6c435e 100644
--- a/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy
+++ b/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy
@@ -19,9 +19,6 @@ import java.util.stream.Collectors
 
 suite("parse_sql_from_sql_cache") {
     def assertHasCache = { String sqlStr ->
-        if (isCloudMode()) {
-            return
-        }
         explain {
             sql ("physical plan ${sqlStr}")
             contains("PhysicalSqlCache")
@@ -29,507 +26,502 @@ suite("parse_sql_from_sql_cache") {
     }
 
     def assertNoCache = { String sqlStr ->
-        if (isCloudMode()) {
-            return
-        }
         explain {
             sql ("physical plan ${sqlStr}")
             notContains("PhysicalSqlCache")
         }
     }
 
-    sql  "ADMIN SET FRONTEND CONFIG ('cache_last_version_interval_second' = '10')"
-
-    combineFutures(
-        extraThread("testUsePlanCache", {
-            createTestTable "test_use_plan_cache"
+    def dbName = (sql "select database()")[0][0].toString()
+    foreachFrontends { fe ->
+        def url = "jdbc:mysql://${fe.Host}:${fe.QueryPort}/${dbName}"
+        connect(context.config.jdbcUser, context.config.jdbcPassword, url) {
+            sql "ADMIN SET FRONTEND CONFIG ('cache_last_version_interval_second' = '10')"
+        }
+    }
 
-            // after partition changed 10s, the sql cache can be used
-            sleep(10000)
+    // make sure if the table has been dropped, the cache should invalidate,
+    // so we should retry multiple times to check
+    for (def __ in 0..3) {
+        combineFutures(
+            extraThread("testUsePlanCache", {
+                createTestTable "test_use_plan_cache"
 
-            sql "set enable_nereids_planner=true"
-            sql "set enable_fallback_to_original_planner=false"
-            sql "set enable_sql_cache=true"
+                // after partition changed 10s, the sql cache can be used
+                sleep(10000)
 
-            assertNoCache "select * from test_use_plan_cache"
+                sql "set enable_nereids_planner=true"
+                sql "set enable_fallback_to_original_planner=false"
+                sql "set enable_sql_cache=true"
 
-            // create sql cache
-            sql "select * from test_use_plan_cache"
+                assertNoCache "select * from test_use_plan_cache"
 
-            // use sql cache
-            assertHasCache "select * from test_use_plan_cache"
-        }),
-        extraThread("testAddPartitionAndInsert", {
-            createTestTable "test_use_plan_cache2"
+                // create sql cache
+                sql "select * from test_use_plan_cache"
 
-            // after partition changed 10s, the sql cache can be used
-            sleep(10000)
+                // use sql cache
+                assertHasCache "select * from test_use_plan_cache"
+            }),
+            extraThread("testAddPartitionAndInsert", {
+                createTestTable "test_use_plan_cache2"
 
-            sql "set enable_nereids_planner=true"
-            sql "set enable_fallback_to_original_planner=false"
-            sql "set enable_sql_cache=true"
+                // after partition changed 10s, the sql cache can be used
+                sleep(10000)
 
-            assertNoCache "select * from test_use_plan_cache2"
-            sql "select * from test_use_plan_cache2"
-            assertHasCache "select * from test_use_plan_cache2"
+                sql "set enable_nereids_planner=true"
+                sql "set enable_fallback_to_original_planner=false"
+                sql "set enable_sql_cache=true"
 
-            // NOTE: in cloud mode, add empty partition can not use cache, because the table version already update,
-            //       but in native mode, add empty partition can use cache
-            sql "alter table test_use_plan_cache2 add partition p6 values[('6'),('7'))"
-            if (isCloudMode()) {
                 assertNoCache "select * from test_use_plan_cache2"
-            } else {
+                sql "select * from test_use_plan_cache2"
                 assertHasCache "select * from test_use_plan_cache2"
-            }
-
-            // insert data can not use cache
-            sql "insert into test_use_plan_cache2 values(6, 1)"
-            assertNoCache "select * from test_use_plan_cache2"
-        }),
-        extraThread("testDropPartition", {
-            createTestTable "test_use_plan_cache3"
-
-            // after partition changed 10s, the sql cache can be used
-            sleep(10000)
-
-            sql "set enable_nereids_planner=true"
-            sql "set enable_fallback_to_original_planner=false"
-            sql "set enable_sql_cache=true"
-
-            assertNoCache "select * from test_use_plan_cache3"
-            sql "select * from test_use_plan_cache3"
-            assertHasCache "select * from test_use_plan_cache3"
-
-            // drop partition can not use cache
-            sql "alter table test_use_plan_cache3 drop partition p5"
-            assertNoCache "select * from test_use_plan_cache3"
-        }),
-        extraThread("testReplacePartition", {
-            createTestTable "test_use_plan_cache4"
-
-            sql "alter table test_use_plan_cache4 add temporary partition tp1 values [('1'), ('2'))"
-
-            streamLoad {
-                table "test_use_plan_cache4"
-                set "temporaryPartitions", "tp1"
-                inputIterator([[1, 3], [1, 4]].iterator())
-            }
-            sql "sync"
-
-            // after partition changed 10s, the sql cache can be used
-            sleep(10000)
-
-            sql "set enable_nereids_planner=true"
-            sql "set enable_fallback_to_original_planner=false"
-            sql "set enable_sql_cache=true"
-
-            assertNoCache "select * from test_use_plan_cache4"
-            sql "select * from test_use_plan_cache4"
-            assertHasCache "select * from test_use_plan_cache4"
-
-            // replace partition can not use cache
-            sql "alter table test_use_plan_cache4 replace partition (p1) with temporary partition(tp1)"
-            assertNoCache "select * from test_use_plan_cache4"
-        }),
-        extraThread("testStreamLoad", {
-            createTestTable "test_use_plan_cache5"
-
-            // after partition changed 10s, the sql cache can be used
-            sleep(10000)
-
-            sql "set enable_nereids_planner=true"
-            sql "set enable_fallback_to_original_planner=false"
-            sql "set enable_sql_cache=true"
-
-            assertNoCache "select * from test_use_plan_cache5"
-            sql "select * from test_use_plan_cache5"
-            assertHasCache "select * from test_use_plan_cache5"
-
-            streamLoad {
-                table "test_use_plan_cache5"
-                set "partitions", "p1"
-                inputIterator([[1, 3], [1, 4]].iterator())
-            }
-            sql "sync"
-
-            // stream load can not use cache
-            sql "select * from test_use_plan_cache5"
-            assertNoCache "select * from test_use_plan_cache5"
-        }),
-        extraThread("testUpdate",{
-            createTestTable("test_use_plan_cache6", true)
-
-            // after partition changed 10s, the sql cache can be used
-            sleep(10000)
-
-            sql "set enable_nereids_planner=true"
-            sql "set enable_fallback_to_original_planner=false"
-            sql "set enable_sql_cache=true"
-
-            assertNoCache "select * from test_use_plan_cache6"
-            sql "select * from test_use_plan_cache6"
-            assertHasCache "select * from test_use_plan_cache6"
-
-            sql "update test_use_plan_cache6 set value=3 where id=1"
-
-            // update can not use cache
-            sql "select * from test_use_plan_cache6"
-            assertNoCache "select * from test_use_plan_cache6"
-        }),
-        extraThread("testDelete", {
-            createTestTable "test_use_plan_cache7"
-
-            // after partition changed 10s, the sql cache can be used
-            sleep(10000)
-
-            sql "set enable_nereids_planner=true"
-            sql "set enable_fallback_to_original_planner=false"
-            sql "set enable_sql_cache=true"
-
-            assertNoCache "select * from test_use_plan_cache7"
-            sql "select * from test_use_plan_cache7"
-            assertHasCache "select * from test_use_plan_cache7"
-
-            sql "delete from test_use_plan_cache7 where id = 1"
-
-            // delete can not use cache
-            sql "select * from test_use_plan_cache7"
-            assertNoCache "select * from test_use_plan_cache7"
-        }),
-        extraThread("testDropTable", {
-            createTestTable "test_use_plan_cache8"
-
-            // after partition changed 10s, the sql cache can be used
-            sleep(10000)
-
-            sql "set enable_nereids_planner=true"
-            sql "set enable_fallback_to_original_planner=false"
-            sql "set enable_sql_cache=true"
-
-            assertNoCache "select * from test_use_plan_cache8"
-            sql "select * from test_use_plan_cache8"
-            assertHasCache "select * from test_use_plan_cache8"
-
-            sql "drop table test_use_plan_cache8"
-
-            // should visible the table has bean deleted
-            test {
-                sql "select * from test_use_plan_cache8"
-                exception "does not exist in database"
-            }
-        }),
-        extraThread("testCreateAndAlterView", {
-            createTestTable "test_use_plan_cache9"
-
-            sql "drop view if exists test_use_plan_cache9_view"
-            sql "create view test_use_plan_cache9_view as select * from test_use_plan_cache9"
-
-            // after partition changed 10s, the sql cache can be used
-            sleep(10000)
-
-            sql "set enable_nereids_planner=true"
-            sql "set enable_fallback_to_original_planner=false"
-            sql "set enable_sql_cache=true"
-
-            assertNoCache "select * from test_use_plan_cache9_view"
-            sql "select * from test_use_plan_cache9_view"
-            assertHasCache "select * from test_use_plan_cache9_view"
-
-            // alter view should not use cache
-            sql "alter view test_use_plan_cache9_view as select id from test_use_plan_cache9"
-            assertNoCache "select * from test_use_plan_cache9_view"
-        }),
-        extraThread("testDropView", {
-            createTestTable "test_use_plan_cache10"
-
-            sql "drop view if exists test_use_plan_cache10_view"
-            sql "create view test_use_plan_cache10_view as select * from test_use_plan_cache10"
-
-            // after partition changed 10s, the sql cache can be used
-            sleep(10000)
-
-            sql "set enable_nereids_planner=true"
-            sql "set enable_fallback_to_original_planner=false"
-            sql "set enable_sql_cache=true"
-
-            assertNoCache "select * from test_use_plan_cache10_view"
-            sql "select * from test_use_plan_cache10_view"
-            assertHasCache "select * from test_use_plan_cache10_view"
-
-            sql "drop view test_use_plan_cache10_view"
-            // should visible the view has bean deleted
-            test {
-                sql "select * from test_use_plan_cache10_view"
-                exception "does not exist in database"
-            }
-        }),
-        extraThread("testBaseTableChanged", {
-            createTestTable "test_use_plan_cache11"
-
-            sql "drop view if exists test_use_plan_cache11_view"
-            sql "create view test_use_plan_cache11_view as select * from test_use_plan_cache11"
-
-            // after partition changed 10s, the sql cache can be used
-            sleep(10000)
-
-            sql "set enable_nereids_planner=true"
-            sql "set enable_fallback_to_original_planner=false"
-            sql "set enable_sql_cache=true"
-
-            assertNoCache "select * from test_use_plan_cache11_view"
-            sql "select * from test_use_plan_cache11_view"
-            assertHasCache "select * from test_use_plan_cache11_view"
-
-            sql "insert into test_use_plan_cache11 values(1, 3)"
-
-            // base table already changed, can not use cache
-            assertNoCache "select * from test_use_plan_cache11_view"
-        }),
-        extraThread("testNotShareCacheBetweenUsers", {
-            sql "drop user if exists test_cache_user1"
-            sql "create user test_cache_user1 identified by 'DORIS@2024'"
-            def dbName = context.config.getDbNameByFile(context.file)
-            sql """GRANT SELECT_PRIV ON *.* TO test_cache_user1"""
-            //cloud-mode
-            if (isCloudMode()) {
-                def clusters = sql " SHOW CLUSTERS; "
-                assertTrue(!clusters.isEmpty())
-                def validCluster = clusters[0][0]
-                sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO test_cache_user1"""
-            }
-
-            createTestTable "test_use_plan_cache12"
-
-            // after partition changed 10s, the sql cache can be used
-            sleep(10000)
-
-            sql "set enable_nereids_planner=true"
-            sql "set enable_fallback_to_original_planner=false"
-            sql "set enable_sql_cache=true"
-
-            assertNoCache "select * from test_use_plan_cache12"
-            sql "select * from test_use_plan_cache12"
-            assertHasCache "select * from test_use_plan_cache12"
-
-            sql "sync"
-
-
-            extraThread("test_cache_user1_thread", {
-                connect("test_cache_user1", "DORIS@2024") {
-                    sql "use ${dbName}"
-                    sql "set enable_nereids_planner=true"
-                    sql "set enable_fallback_to_original_planner=false"
-                    sql "set enable_sql_cache=true"
 
-                    assertNoCache "select * from test_use_plan_cache12"
+                // NOTE: in cloud mode, add empty partition can not use cache, because the table version already update,
+                //       but in native mode, add empty partition can use cache
+                sql "alter table test_use_plan_cache2 add partition p6 values[('6'),('7'))"
+                if (isCloudMode()) {
+                    assertNoCache "select * from test_use_plan_cache2"
+                } else {
+                    assertHasCache "select * from test_use_plan_cache2"
                 }
-            }).get()
-        }),
-        extraThread("testAddRowPolicy", {
-            def dbName = context.config.getDbNameByFile(context.file)
-            try_sql """
-                DROP ROW POLICY if exists test_cache_row_policy_2
-                ON ${dbName}.test_use_plan_cache13
-                FOR test_cache_user2"""
-
-            sql "drop user if exists test_cache_user2"
-            sql "create user test_cache_user2 identified by 'DORIS@2024'"
-            sql """GRANT SELECT_PRIV ON *.* TO test_cache_user2"""
-            //cloud-mode
-            if (isCloudMode()) {
-                def clusters = sql " SHOW CLUSTERS; "
-                assertTrue(!clusters.isEmpty())
-                def validCluster = clusters[0][0]
-                sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO test_cache_user2"""
-            }
-
-            createTestTable "test_use_plan_cache13"
-
-            // after partition changed 10s, the sql cache can be used
-            sleep(10000)
-
-            sql "sync"
-
-            extraThread("test_cache_user2_thread", {
-                connect("test_cache_user2", "DORIS@2024") {
-                    sql "use ${dbName}"
-                    sql "set enable_nereids_planner=true"
-                    sql "set enable_fallback_to_original_planner=false"
-                    sql "set enable_sql_cache=true"
 
-                    assertNoCache "select * from test_use_plan_cache13"
-                    sql "select * from test_use_plan_cache13"
-                    assertHasCache "select * from test_use_plan_cache13"
+                // insert data can not use cache
+                sql "insert into test_use_plan_cache2 values(6, 1)"
+                assertNoCache "select * from test_use_plan_cache2"
+            }),
+            extraThread("testDropPartition", {
+                createTestTable "test_use_plan_cache3"
+
+                // after partition changed 10s, the sql cache can be used
+                sleep(10000)
+
+                sql "set enable_nereids_planner=true"
+                sql "set enable_fallback_to_original_planner=false"
+                sql "set enable_sql_cache=true"
+
+                assertNoCache "select * from test_use_plan_cache3"
+                sql "select * from test_use_plan_cache3"
+                assertHasCache "select * from test_use_plan_cache3"
+
+                // drop partition can not use cache
+                sql "alter table test_use_plan_cache3 drop partition p5"
+                assertNoCache "select * from test_use_plan_cache3"
+            }),
+            extraThread("testReplacePartition", {
+                createTestTable "test_use_plan_cache4"
+
+                sql "alter table test_use_plan_cache4 add temporary partition tp1 values [('1'), ('2'))"
+
+                streamLoad {
+                    table "test_use_plan_cache4"
+                    set "temporaryPartitions", "tp1"
+                    inputIterator([[1, 3], [1, 4]].iterator())
                 }
-            }).get()
+                sql "sync"
 
-            sql """
-                CREATE ROW POLICY test_cache_row_policy_2
-                ON ${dbName}.test_use_plan_cache13
-                AS RESTRICTIVE TO test_cache_user2
-                USING (id = 4)"""
+                // after partition changed 10s, the sql cache can be used
+                sleep(10000)
 
-            sql "sync"
+                sql "set enable_nereids_planner=true"
+                sql "set enable_fallback_to_original_planner=false"
+                sql "set enable_sql_cache=true"
 
-            // after row policy changed, the cache is invalidate
-            extraThread("test_cache_user2_thread2", {
-                connect("test_cache_user2", "DORIS@2024") {
-                    sql "use ${dbName}"
-                    sql "set enable_nereids_planner=true"
-                    sql "set enable_fallback_to_original_planner=false"
-                    sql "set enable_sql_cache=true"
+                assertNoCache "select * from test_use_plan_cache4"
+                sql "select * from test_use_plan_cache4"
+                assertHasCache "select * from test_use_plan_cache4"
+
+                // replace partition can not use cache
+                sql "alter table test_use_plan_cache4 replace partition (p1) with temporary partition(tp1)"
+                assertNoCache "select * from test_use_plan_cache4"
+            }),
+            extraThread("testStreamLoad", {
+                createTestTable "test_use_plan_cache5"
+
+                // after partition changed 10s, the sql cache can be used
+                sleep(10000)
 
-                    assertNoCache "select * from test_use_plan_cache13"
+                sql "set enable_nereids_planner=true"
+                sql "set enable_fallback_to_original_planner=false"
+                sql "set enable_sql_cache=true"
+
+                assertNoCache "select * from test_use_plan_cache5"
+                sql "select * from test_use_plan_cache5"
+                assertHasCache "select * from test_use_plan_cache5"
+
+                streamLoad {
+                    table "test_use_plan_cache5"
+                    set "partitions", "p1"
+                    inputIterator([[1, 3], [1, 4]].iterator())
                 }
-            }).get()
-        }),
-        extraThread("testDropRowPolicy", {
-            def dbName = context.config.getDbNameByFile(context.file)
-            try_sql """
-            DROP ROW POLICY if exists test_cache_row_policy_3
-            ON ${dbName}.test_use_plan_cache14
-            FOR test_cache_user3"""
-
-            sql "drop user if exists test_cache_user3"
-            sql "create user test_cache_user3 identified by 'DORIS@2024'"
-            sql """GRANT SELECT_PRIV ON *.* TO test_cache_user3"""
-            //cloud-mode
-            if (isCloudMode()) {
-                def clusters = sql " SHOW CLUSTERS; "
-                assertTrue(!clusters.isEmpty())
-                def validCluster = clusters[0][0]
-                sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO test_cache_user3"""
-            }
-
-            createTestTable "test_use_plan_cache14"
-
-            sql """
-            CREATE ROW POLICY test_cache_row_policy_3
-            ON ${dbName}.test_use_plan_cache14
-            AS RESTRICTIVE TO test_cache_user3
-            USING (id = 4)"""
-
-            sql "sync"
-
-            // after partition changed 10s, the sql cache can be used
-            sleep(10000)
-
-            extraThread("test_cache_user3_thread", {
-                connect("test_cache_user3", "DORIS@2024") {
-                    sql "use ${dbName}"
-                    sql "set enable_nereids_planner=true"
-                    sql "set enable_fallback_to_original_planner=false"
-                    sql "set enable_sql_cache=true"
+                sql "sync"
+
+                // stream load can not use cache
+                sql "select * from test_use_plan_cache5"
+                assertNoCache "select * from test_use_plan_cache5"
+            }),
+            extraThread("testUpdate",{
+                createTestTable("test_use_plan_cache6", true)
 
-                    assertNoCache "select * from test_use_plan_cache14"
-                    sql "select * from test_use_plan_cache14"
-                    assertHasCache "select * from test_use_plan_cache14"
+                // after partition changed 10s, the sql cache can be used
+                sleep(10000)
+
+                sql "set enable_nereids_planner=true"
+                sql "set enable_fallback_to_original_planner=false"
+                sql "set enable_sql_cache=true"
+
+                assertNoCache "select * from test_use_plan_cache6"
+                sql "select * from test_use_plan_cache6"
+                assertHasCache "select * from test_use_plan_cache6"
+
+                sql "update test_use_plan_cache6 set value=3 where id=1"
+
+                // update can not use cache
+                sql "select * from test_use_plan_cache6"
+                assertNoCache "select * from test_use_plan_cache6"
+            }),
+            extraThread("testDelete", {
+                createTestTable "test_use_plan_cache7"
+
+                // after partition changed 10s, the sql cache can be used
+                sleep(10000)
+
+                sql "set enable_nereids_planner=true"
+                sql "set enable_fallback_to_original_planner=false"
+                sql "set enable_sql_cache=true"
+
+                assertNoCache "select * from test_use_plan_cache7"
+                sql "select * from test_use_plan_cache7"
+                assertHasCache "select * from test_use_plan_cache7"
+
+                sql "delete from test_use_plan_cache7 where id = 1"
+
+                // delete can not use cache
+                sql "select * from test_use_plan_cache7"
+                assertNoCache "select * from test_use_plan_cache7"
+            }),
+            extraThread("testDropTable", {
+                createTestTable "test_use_plan_cache8"
+
+                // after partition changed 10s, the sql cache can be used
+                sleep(10000)
+
+                sql "set enable_nereids_planner=true"
+                sql "set enable_fallback_to_original_planner=false"
+                sql "set enable_sql_cache=true"
+
+                assertNoCache "select * from test_use_plan_cache8"
+                sql "select * from test_use_plan_cache8"
+                assertHasCache "select * from test_use_plan_cache8"
+
+                sql "drop table test_use_plan_cache8"
+
+                // should visible the table has bean deleted
+                test {
+                    sql "select * from test_use_plan_cache8"
+                    exception "does not exist in database"
                 }
-            }).get()
+            }),
+            extraThread("testCreateAndAlterView", {
+                createTestTable "test_use_plan_cache9"
 
-            try_sql """
-            DROP ROW POLICY if exists test_cache_row_policy_3
-            ON ${dbName}.test_use_plan_cache14
-            FOR test_cache_user3"""
+                sql "drop view if exists test_use_plan_cache9_view"
+                sql "create view test_use_plan_cache9_view as select * from test_use_plan_cache9"
 
-            sql "sync"
+                // after partition changed 10s, the sql cache can be used
+                sleep(10000)
 
-            // after row policy changed, the cache is invalidate
-            extraThread("test_cache_user3_thread2", {
-                connect("test_cache_user3", "DORIS@2024") {
-                    sql "use ${dbName}"
-                    sql "set enable_nereids_planner=true"
-                    sql "set enable_fallback_to_original_planner=false"
-                    sql "set enable_sql_cache=true"
+                sql "set enable_nereids_planner=true"
+                sql "set enable_fallback_to_original_planner=false"
+                sql "set enable_sql_cache=true"
+
+                assertNoCache "select * from test_use_plan_cache9_view"
+                sql "select * from test_use_plan_cache9_view"
+                assertHasCache "select * from test_use_plan_cache9_view"
+
+                // alter view should not use cache
+                sql "alter view test_use_plan_cache9_view as select id from test_use_plan_cache9"
+                assertNoCache "select * from test_use_plan_cache9_view"
+            }),
+            extraThread("testDropView", {
+                createTestTable "test_use_plan_cache10"
 
-                    assertNoCache "select * from test_use_plan_cache14"
+                sql "drop view if exists test_use_plan_cache10_view"
+                sql "create view test_use_plan_cache10_view as select * from test_use_plan_cache10"
+
+                // after partition changed 10s, the sql cache can be used
+                sleep(10000)
+
+                sql "set enable_nereids_planner=true"
+                sql "set enable_fallback_to_original_planner=false"
+                sql "set enable_sql_cache=true"
+
+                assertNoCache "select * from test_use_plan_cache10_view"
+                sql "select * from test_use_plan_cache10_view"
+                assertHasCache "select * from test_use_plan_cache10_view"
+
+                sql "drop view test_use_plan_cache10_view"
+                // should visible the view has bean deleted
+                test {
+                    sql "select * from test_use_plan_cache10_view"
+                    exception "does not exist in database"
                 }
-            }).get()
-        }),
-        extraThread("testRemovePrivilege", {
-            def dbName = context.config.getDbNameByFile(context.file)
-
-            createTestTable "test_use_plan_cache15"
-
-            // after partition changed 10s, the sql cache can be used
-            sleep(10000)
-
-            sql "drop user if exists test_cache_user4"
-            sql "create user test_cache_user4 identified by 'DORIS@2024'"
-            sql "GRANT SELECT_PRIV ON regression_test.* TO test_cache_user4"
-            sql "GRANT SELECT_PRIV ON ${dbName}.test_use_plan_cache15 TO test_cache_user4"
-            //cloud-mode
-            if (isCloudMode()) {
-                def clusters = sql " SHOW CLUSTERS; "
-                assertTrue(!clusters.isEmpty())
-                def validCluster = clusters[0][0]
-                sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO test_cache_user4"""
-            }
-
-            sql "sync"
-
-            extraThread("test_cache_user4_thread", {
-                connect("test_cache_user4", "DORIS@2024") {
-                    sql "use ${dbName}"
-                    sql "set enable_nereids_planner=true"
-                    sql "set enable_fallback_to_original_planner=false"
-                    sql "set enable_sql_cache=true"
+            }),
+            extraThread("testBaseTableChanged", {
+                createTestTable "test_use_plan_cache11"
 
-                    assertNoCache "select * from test_use_plan_cache15"
-                    sql "select * from test_use_plan_cache15"
-                    assertHasCache "select * from test_use_plan_cache15"
+                sql "drop view if exists test_use_plan_cache11_view"
+                sql "create view test_use_plan_cache11_view as select * from test_use_plan_cache11"
+
+                // after partition changed 10s, the sql cache can be used
+                sleep(10000)
+
+                sql "set enable_nereids_planner=true"
+                sql "set enable_fallback_to_original_planner=false"
+                sql "set enable_sql_cache=true"
+
+                assertNoCache "select * from test_use_plan_cache11_view"
+                sql "select * from test_use_plan_cache11_view"
+                assertHasCache "select * from test_use_plan_cache11_view"
+
+                sql "insert into test_use_plan_cache11 values(1, 3)"
+
+                // base table already changed, can not use cache
+                assertNoCache "select * from test_use_plan_cache11_view"
+            }),
+            extraThread("testNotShareCacheBetweenUsers", {
+                sql "drop user if exists test_cache_user1"
+                sql "create user test_cache_user1 identified by 'DORIS@2024'"
+                sql """GRANT SELECT_PRIV ON *.* TO test_cache_user1"""
+                //cloud-mode
+                if (isCloudMode()) {
+                    def clusters = sql " SHOW CLUSTERS; "
+                    assertTrue(!clusters.isEmpty())
+                    def validCluster = clusters[0][0]
+                    sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO test_cache_user1"""
                 }
-            }).get()
 
-            sql "REVOKE SELECT_PRIV ON ${dbName}.test_use_plan_cache15 FROM test_cache_user4"
+                createTestTable "test_use_plan_cache12"
 
-            sql "sync"
+                // after partition changed 10s, the sql cache can be used
+                sleep(10000)
 
-            // after privileges changed, the cache is invalidate
-            extraThread("test_cache_user4_thread2", {
-                connect("test_cache_user4", "DORIS@2024") {
-                    sql "set enable_nereids_planner=true"
-                    sql "set enable_fallback_to_original_planner=false"
-                    sql "set enable_sql_cache=true"
+                sql "set enable_nereids_planner=true"
+                sql "set enable_fallback_to_original_planner=false"
+                sql "set enable_sql_cache=true"
 
-                    test {
-                        sql ("select * from ${dbName}.test_use_plan_cache15")
-                        exception "Permission denied"
+                assertNoCache "select * from test_use_plan_cache12"
+                sql "select * from test_use_plan_cache12"
+                assertHasCache "select * from test_use_plan_cache12"
+
+                sql "sync"
+
+                extraThread("test_cache_user1_thread", {
+                    connect("test_cache_user1", "DORIS@2024") {
+                        sql "use ${dbName}"
+                        sql "set enable_nereids_planner=true"
+                        sql "set enable_fallback_to_original_planner=false"
+                        sql "set enable_sql_cache=true"
+
+                        assertNoCache "select * from test_use_plan_cache12"
+                    }
+                }).get()
+            }),
+            extraThread("testAddRowPolicy", {
+                try_sql """
+                    DROP ROW POLICY if exists test_cache_row_policy_2
+                    ON ${dbName}.test_use_plan_cache13
+                    FOR test_cache_user2"""
+
+                sql "drop user if exists test_cache_user2"
+                sql "create user test_cache_user2 identified by 'DORIS@2024'"
+                sql """GRANT SELECT_PRIV ON *.* TO test_cache_user2"""
+                //cloud-mode
+                if (isCloudMode()) {
+                    def clusters = sql " SHOW CLUSTERS; "
+                    assertTrue(!clusters.isEmpty())
+                    def validCluster = clusters[0][0]
+                    sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO test_cache_user2"""
+                }
+
+                createTestTable "test_use_plan_cache13"
+
+                // after partition changed 10s, the sql cache can be used
+                sleep(10000)
+
+                sql "sync"
+
+                extraThread("test_cache_user2_thread", {
+                    connect("test_cache_user2", "DORIS@2024") {
+                        sql "use ${dbName}"
+                        sql "set enable_nereids_planner=true"
+                        sql "set enable_fallback_to_original_planner=false"
+                        sql "set enable_sql_cache=true"
+
+                        assertNoCache "select * from test_use_plan_cache13"
+                        sql "select * from test_use_plan_cache13"
+                        assertHasCache "select * from test_use_plan_cache13"
+                    }
+                }).get()
+
+                sql """
+                    CREATE ROW POLICY test_cache_row_policy_2
+                    ON ${dbName}.test_use_plan_cache13
+                    AS RESTRICTIVE TO test_cache_user2
+                    USING (id = 4)"""
+
+                sql "sync"
+
+                // after row policy changed, the cache is invalidate
+                extraThread("test_cache_user2_thread2", {
+                    connect("test_cache_user2", "DORIS@2024") {
+                        sql "use ${dbName}"
+                        sql "set enable_nereids_planner=true"
+                        sql "set enable_fallback_to_original_planner=false"
+                        sql "set enable_sql_cache=true"
+
+                        assertNoCache "select * from test_use_plan_cache13"
                     }
+                }).get()
+            }),
+            extraThread("testDropRowPolicy", {
+                try_sql """
+                DROP ROW POLICY if exists test_cache_row_policy_3
+                ON ${dbName}.test_use_plan_cache14
+                FOR test_cache_user3"""
+
+                sql "drop user if exists test_cache_user3"
+                sql "create user test_cache_user3 identified by 'DORIS@2024'"
+                sql """GRANT SELECT_PRIV ON *.* TO test_cache_user3"""
+                //cloud-mode
+                if (isCloudMode()) {
+                    def clusters = sql " SHOW CLUSTERS; "
+                    assertTrue(!clusters.isEmpty())
+                    def validCluster = clusters[0][0]
+                    sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO test_cache_user3"""
                 }
-            }).get()
-        }),
-        extraThread("testNondeterministic", {
-            createTestTable "test_use_plan_cache16"
-
-            // after partition changed 10s, the sql cache can be used
-            sleep(10000)
-
-            sql "set enable_nereids_planner=true"
-            sql "set enable_fallback_to_original_planner=false"
-            sql "set enable_sql_cache=true"
-
-            assertNoCache "select random() from test_use_plan_cache16"
-            // create sql cache
-            sql "select random() from test_use_plan_cache16"
-            // can not use sql cache
-            assertNoCache "select random() from test_use_plan_cache16"
-
-
-            assertNoCache "select year(now()) from test_use_plan_cache16"
-            sql "select year(now()) from test_use_plan_cache16"
-            assertHasCache "select year(now()) from test_use_plan_cache16"
-
-
-            assertNoCache "select second(now()) from test_use_plan_cache16"
-            sql "select second(now()) from test_use_plan_cache16"
-            sleep(1000)
-            assertNoCache "select second(now()) from test_use_plan_cache16"
-        }),
-        extraThread("testUserVariable", {
-            // make sure if the table has been dropped, the cache should invalidate,
-            // so we should retry twice to check
-            for (def i in 0..2) {
+
+                createTestTable "test_use_plan_cache14"
+
+                sql """
+                CREATE ROW POLICY test_cache_row_policy_3
+                ON ${dbName}.test_use_plan_cache14
+                AS RESTRICTIVE TO test_cache_user3
+                USING (id = 4)"""
+
+                sql "sync"
+
+                // after partition changed 10s, the sql cache can be used
+                sleep(10000)
+
+                extraThread("test_cache_user3_thread", {
+                    connect("test_cache_user3", "DORIS@2024") {
+                        sql "use ${dbName}"
+                        sql "set enable_nereids_planner=true"
+                        sql "set enable_fallback_to_original_planner=false"
+                        sql "set enable_sql_cache=true"
+
+                        assertNoCache "select * from test_use_plan_cache14"
+                        sql "select * from test_use_plan_cache14"
+                        assertHasCache "select * from test_use_plan_cache14"
+                    }
+                }).get()
+
+                try_sql """
+                DROP ROW POLICY if exists test_cache_row_policy_3
+                ON ${dbName}.test_use_plan_cache14
+                FOR test_cache_user3"""
+
+                sql "sync"
+
+                // after row policy changed, the cache is invalidate
+                extraThread("test_cache_user3_thread2", {
+                    connect("test_cache_user3", "DORIS@2024") {
+                        sql "use ${dbName}"
+                        sql "set enable_nereids_planner=true"
+                        sql "set enable_fallback_to_original_planner=false"
+                        sql "set enable_sql_cache=true"
+
+                        assertNoCache "select * from test_use_plan_cache14"
+                    }
+                }).get()
+            }),
+            extraThread("testRemovePrivilege", {
+                createTestTable "test_use_plan_cache15"
+
+                // after partition changed 10s, the sql cache can be used
+                sleep(10000)
+
+                sql "drop user if exists test_cache_user4"
+                sql "create user test_cache_user4 identified by 'DORIS@2024'"
+                sql "GRANT SELECT_PRIV ON regression_test.* TO test_cache_user4"
+                sql "GRANT SELECT_PRIV ON ${dbName}.test_use_plan_cache15 TO test_cache_user4"
+                //cloud-mode
+                if (isCloudMode()) {
+                    def clusters = sql " SHOW CLUSTERS; "
+                    assertTrue(!clusters.isEmpty())
+                    def validCluster = clusters[0][0]
+                    sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO test_cache_user4"""
+                }
+
+                sql "sync"
+
+                extraThread("test_cache_user4_thread", {
+                    connect("test_cache_user4", "DORIS@2024") {
+                        sql "use ${dbName}"
+                        sql "set enable_nereids_planner=true"
+                        sql "set enable_fallback_to_original_planner=false"
+                        sql "set enable_sql_cache=true"
+
+                        assertNoCache "select * from test_use_plan_cache15"
+                        sql "select * from test_use_plan_cache15"
+                        assertHasCache "select * from test_use_plan_cache15"
+                    }
+                }).get()
+
+                sql "REVOKE SELECT_PRIV ON ${dbName}.test_use_plan_cache15 FROM test_cache_user4"
+
+                sql "sync"
+
+                // after privileges changed, the cache is invalidate
+                extraThread("test_cache_user4_thread2", {
+                    connect("test_cache_user4", "DORIS@2024") {
+                        sql "set enable_nereids_planner=true"
+                        sql "set enable_fallback_to_original_planner=false"
+                        sql "set enable_sql_cache=true"
+
+                        test {
+                            sql ("select * from ${dbName}.test_use_plan_cache15")
+                            exception "Permission denied"
+                        }
+                    }
+                }).get()
+            }),
+            extraThread("testNondeterministic", {
+                createTestTable "test_use_plan_cache16"
+
+                // after partition changed 10s, the sql cache can be used
+                sleep(10000)
+
+                sql "set enable_nereids_planner=true"
+                sql "set enable_fallback_to_original_planner=false"
+                sql "set enable_sql_cache=true"
+
+                assertNoCache "select random() from test_use_plan_cache16"
+                // create sql cache
+                sql "select random() from test_use_plan_cache16"
+                // can not use sql cache
+                assertNoCache "select random() from test_use_plan_cache16"
+
+                assertNoCache "select year(now()) from test_use_plan_cache16"
+                sql "select year(now()) from test_use_plan_cache16"
+                assertHasCache "select year(now()) from test_use_plan_cache16"
+
+                assertNoCache "select second(now()) from test_use_plan_cache16"
+                sql "select second(now()) from test_use_plan_cache16"
+                sleep(1000)
+                assertNoCache "select second(now()) from test_use_plan_cache16"
+            }),
+            extraThread("testUserVariable", {
                 createTestTable "test_use_plan_cache17"
 
                 // after partition changed 10s, the sql cache can be used
@@ -559,7 +551,6 @@ suite("parse_sql_from_sql_cache") {
                 def result1 = sql "select @custom_variable from test_use_plan_cache17 where id = 1 and value = 1"
                 assertTrue(result1.size() == 1 && result1[0][0].toString().toInteger() == 10)
 
-
                 sql "set @custom_variable2=1"
                 assertNoCache "select * from test_use_plan_cache17 where id = @custom_variable2 and value = 1"
                 def res = sql "select * from test_use_plan_cache17 where id = @custom_variable2 and value = 1"
@@ -578,81 +569,106 @@ suite("parse_sql_from_sql_cache") {
                 assertHasCache "select * from test_use_plan_cache17 where id = @custom_variable2 and value = 1"
                 res = sql "select * from test_use_plan_cache17 where id = @custom_variable2 and value = 1"
                 assertTrue(res[0][0] == 1)
-            }
-        }),
-        extraThread("test_udf", {
-            def jarPath = """${context.config.suitePath}/javaudf_p0/jars/java-udf-case-jar-with-dependencies.jar"""
-            scp_udf_file_to_all_be(jarPath)
-            try_sql("DROP FUNCTION IF EXISTS java_udf_string_test(string, int, int);")
-            try_sql("DROP TABLE IF EXISTS test_javaudf_string")
-
-            sql """ DROP TABLE IF EXISTS test_javaudf_string """
-            sql """
-                    CREATE TABLE IF NOT EXISTS test_javaudf_string (
-                        `user_id`     INT         NOT NULL COMMENT "用户id",
-                        `char_col`    CHAR        NOT NULL COMMENT "",
-                        `varchar_col` VARCHAR(10) NOT NULL COMMENT "",
-                        `string_col`  STRING      NOT NULL COMMENT ""
-                        )
-                        DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
-                    """
-
-            StringBuilder values = new StringBuilder()
-            int i = 1
-            for (; i < 9; i ++) {
-                values.append(" (${i}, '${i}','abcdefg${i}','poiuytre${i}abcdefg'),\n")
-            }
-            values.append("(${i}, '${i}','abcdefg${i}','poiuytre${i}abcdefg')")
-
-            sql "INSERT INTO test_javaudf_string VALUES ${values}"
-            sql "sync"
-
-            File path = new File(jarPath)
-            if (!path.exists()) {
-                throw new IllegalStateException("""${jarPath} doesn't exist! """)
-            }
-
-            sql """ CREATE FUNCTION java_udf_string_test(string, int, int) RETURNS string PROPERTIES (
-                        "file"="file://${jarPath}",
-                        "symbol"="org.apache.doris.udf.StringTest",
-                        "type"="JAVA_UDF"
-                    ); """
-
-            assertNoCache "SELECT java_udf_string_test(varchar_col, 2, 3) result FROM test_javaudf_string ORDER BY result;"
-            sql "SELECT java_udf_string_test(varchar_col, 2, 3) result FROM test_javaudf_string ORDER BY result;"
-            assertNoCache "SELECT java_udf_string_test(varchar_col, 2, 3) result FROM test_javaudf_string ORDER BY result;"
-        }),
-        extraThread("testMultiFrontends", {
-            def aliveFrontends = sql_return_maparray("show frontends")
-                    .stream()
-                    .filter { it["Alive"].toString().equalsIgnoreCase("true") }
-                    .collect(Collectors.toList())
-
-            if (aliveFrontends.size() <= 1) {
-                return
-            }
-
-            def fe1 = aliveFrontends[0]["Host"] + ":" + aliveFrontends[0]["QueryPort"]
-            def fe2 = fe1
-            if (aliveFrontends.size() > 1) {
-                fe2 = aliveFrontends[1]["Host"] + ":" + aliveFrontends[1]["QueryPort"]
-            }
-
-            log.info("fe1: ${fe1}")
-            log.info("fe2: ${fe2}")
-
-            def dbName = context.config.getDbNameByFile(context.file)
-
-            log.info("connect to fe: ${fe1}")
-            connect( context.config.jdbcUser,  context.config.jdbcPassword,  "jdbc:mysql://${fe1}") {
-                sql  "ADMIN SET FRONTEND CONFIG ('cache_last_version_interval_second' = '10')"
-
-                sql "use ${dbName}"
-
-                createTestTable "test_use_plan_cache18"
+            }),
+            extraThread("test_udf", {
+                def jarPath = """${context.config.suitePath}/javaudf_p0/jars/java-udf-case-jar-with-dependencies.jar"""
+                scp_udf_file_to_all_be(jarPath)
+                try_sql("DROP FUNCTION IF EXISTS java_udf_string_test(string, int, int);")
+                try_sql("DROP TABLE IF EXISTS test_javaudf_string")
+
+                sql """ DROP TABLE IF EXISTS test_javaudf_string """
+                sql """
+                        CREATE TABLE IF NOT EXISTS test_javaudf_string (
+                            `user_id`     INT         NOT NULL COMMENT "用户id",
+                            `char_col`    CHAR        NOT NULL COMMENT "",
+                            `varchar_col` VARCHAR(10) NOT NULL COMMENT "",
+                            `string_col`  STRING      NOT NULL COMMENT ""
+                            )
+                            DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
+                        """
+
+                StringBuilder values = new StringBuilder()
+                int i = 1
+                for (; i < 9; i ++) {
+                    values.append(" (${i}, '${i}','abcdefg${i}','poiuytre${i}abcdefg'),\n")
+                }
+                values.append("(${i}, '${i}','abcdefg${i}','poiuytre${i}abcdefg')")
 
+                sql "INSERT INTO test_javaudf_string VALUES ${values}"
                 sql "sync"
 
+                sleep(10000)
+
+                File path = new File(jarPath)
+                if (!path.exists()) {
+                    throw new IllegalStateException("""${jarPath} doesn't exist! """)
+                }
+
+                sql """ CREATE FUNCTION java_udf_string_test(string, int, int) RETURNS string PROPERTIES (
+                            "file"="file://${jarPath}",
+                            "symbol"="org.apache.doris.udf.StringTest",
+                            "type"="JAVA_UDF"
+                        ); """
+
+                assertNoCache "SELECT java_udf_string_test(varchar_col, 2, 3) result FROM test_javaudf_string ORDER BY result;"
+                sql "SELECT java_udf_string_test(varchar_col, 2, 3) result FROM test_javaudf_string ORDER BY result;"
+                assertNoCache "SELECT java_udf_string_test(varchar_col, 2, 3) result FROM test_javaudf_string ORDER BY result;"
+            }),
+            extraThread("testMultiFrontends", {
+                def aliveFrontends = sql_return_maparray("show frontends")
+                        .stream()
+                        .filter { it["Alive"].toString().equalsIgnoreCase("true") }
+                        .collect(Collectors.toList())
+
+                if (aliveFrontends.size() <= 1) {
+                    return
+                }
+
+                def fe1 = aliveFrontends[0]["Host"] + ":" + aliveFrontends[0]["QueryPort"]
+                def fe2 = fe1
+                if (aliveFrontends.size() > 1) {
+                    fe2 = aliveFrontends[1]["Host"] + ":" + aliveFrontends[1]["QueryPort"]
+                }
+
+                log.info("fe1: ${fe1}")
+                log.info("fe2: ${fe2}")
+
+                log.info("connect to fe: ${fe1}")
+                connect( context.config.jdbcUser,  context.config.jdbcPassword,  "jdbc:mysql://${fe1}") {
+                    sql "use ${dbName}"
+
+                    createTestTable "test_use_plan_cache18"
+
+                    sql "sync"
+
+                    // after partition changed 10s, the sql cache can be used
+                    sleep(10000)
+
+                    sql "set enable_nereids_planner=true"
+                    sql "set enable_fallback_to_original_planner=false"
+                    sql "set enable_sql_cache=true"
+
+                    assertNoCache "select * from test_use_plan_cache18"
+                    sql "select * from test_use_plan_cache18"
+                    assertHasCache "select * from test_use_plan_cache18"
+                }
+
+                log.info("connect to fe: ${fe2}")
+                connect( context.config.jdbcUser,  context.config.jdbcPassword,  "jdbc:mysql://${fe2}") {
+
+                    sql "use ${dbName}"
+                    sql "set enable_nereids_planner=true"
+                    sql "set enable_fallback_to_original_planner=false"
+                    sql "set enable_sql_cache=true"
+
+                    assertNoCache "select * from test_use_plan_cache18"
+                    sql "select * from test_use_plan_cache18"
+                    assertHasCache "select * from test_use_plan_cache18"
+                }
+            }),
+            extraThread("test_dry_run_query", {
+                createTestTable "test_use_plan_cache19"
+
                 // after partition changed 10s, the sql cache can be used
                 sleep(10000)
 
@@ -660,253 +676,225 @@ suite("parse_sql_from_sql_cache") {
                 sql "set enable_fallback_to_original_planner=false"
                 sql "set enable_sql_cache=true"
 
-                assertNoCache "select * from test_use_plan_cache18"
-                sql "select * from test_use_plan_cache18"
-                assertHasCache "select * from test_use_plan_cache18"
-            }
+                sql "set dry_run_query=true"
+                assertNoCache "select * from test_use_plan_cache19 order by 1, 2"
+                def result1 = sql "select * from test_use_plan_cache19 order by 1, 2"
+                assertTrue(result1.size() == 1)
+                assertNoCache "select * from test_use_plan_cache19 order by 1, 2"
+
+                sql "set dry_run_query=false"
+                assertNoCache "select * from test_use_plan_cache19 order by 1, 2"
+                def result2 = sql "select * from test_use_plan_cache19 order by 1, 2"
+                assertTrue(result2.size() > 1)
+                assertHasCache "select * from test_use_plan_cache19 order by 1, 2"
+
+                sql "set dry_run_query=true"
+                assertNoCache "select * from test_use_plan_cache19 order by 1, 2"
+                def result3 = sql "select * from test_use_plan_cache19 order by 1, 2"
+                assertTrue(result3.size() == 1)
+                assertNoCache "select * from test_use_plan_cache19 order by 1, 2"
+            }),
+            extraThread("test_sql_cache_in_fe", {
+                createTestTable "test_use_plan_cache20"
+
+                sql "alter table test_use_plan_cache20 add partition p6 values[('999'), ('1000'))"
 
-            log.info("connect to fe: ${fe2}")
-            connect( context.config.jdbcUser,  context.config.jdbcPassword,  "jdbc:mysql://${fe2}") {
-                sql  "ADMIN SET FRONTEND CONFIG ('cache_last_version_interval_second' = '10')"
+                // after partition changed 10s, the sql cache can be used
+                sleep(10000)
 
-                sql "use ${dbName}"
                 sql "set enable_nereids_planner=true"
                 sql "set enable_fallback_to_original_planner=false"
                 sql "set enable_sql_cache=true"
 
-                assertNoCache "select * from test_use_plan_cache18"
-                sql "select * from test_use_plan_cache18"
-                assertHasCache "select * from test_use_plan_cache18"
-            }
-        }),
-        extraThread("test_dry_run_query", {
-            createTestTable "test_use_plan_cache19"
-
-            // after partition changed 10s, the sql cache can be used
-            sleep(10000)
-
-            sql "set enable_nereids_planner=true"
-            sql "set enable_fallback_to_original_planner=false"
-            sql "set enable_sql_cache=true"
-
-            sql "set dry_run_query=true"
-            assertNoCache "select * from test_use_plan_cache19 order by 1, 2"
-            def result1 = sql "select * from test_use_plan_cache19 order by 1, 2"
-            assertTrue(result1.size() == 1)
-            assertNoCache "select * from test_use_plan_cache19 order by 1, 2"
-
-            sql "set dry_run_query=false"
-            assertNoCache "select * from test_use_plan_cache19 order by 1, 2"
-            def result2 = sql "select * from test_use_plan_cache19 order by 1, 2"
-            assertTrue(result2.size() > 1)
-            assertHasCache "select * from test_use_plan_cache19 order by 1, 2"
-
-            sql "set dry_run_query=true"
-            assertNoCache "select * from test_use_plan_cache19 order by 1, 2"
-            def result3 = sql "select * from test_use_plan_cache19 order by 1, 2"
-            assertTrue(result3.size() == 1)
-            assertNoCache "select * from test_use_plan_cache19 order by 1, 2"
-        }),
-        extraThread("test_sql_cache_in_fe", {
-            createTestTable "test_use_plan_cache20"
-
-            sql "alter table test_use_plan_cache20 add partition p6 values[('999'), ('1000'))"
-
-            // after partition changed 10s, the sql cache can be used
-            sleep(10000)
-
-            sql "set enable_nereids_planner=true"
-            sql "set enable_fallback_to_original_planner=false"
-            sql "set enable_sql_cache=true"
-
-            int randomInt = Math.random() * 2000000000
-
-            assertNoCache "select * from (select $randomInt as id)a"
-            def result1 = sql "select * from (select $randomInt as id)a"
-            assertTrue(result1.size() == 1)
-
-            assertHasCache "select * from (select $randomInt as id)a"
-            def result2 = sql "select * from (select $randomInt as id)a"
-            assertTrue(result2.size() == 1)
-
-            sql "select * from test_use_plan_cache20 limit 0"
-            assertHasCache "select * from test_use_plan_cache20 limit 0"
-            def result4 = sql "select * from test_use_plan_cache20 limit 0"
-            assertTrue(result4.isEmpty())
-
-            assertNoCache "select * from test_use_plan_cache20 where id=999"
-            def result5 = sql "select * from test_use_plan_cache20 where id=999"
-            assertTrue(result5.isEmpty())
-            assertHasCache "select * from test_use_plan_cache20 where id=999"
-            def result6 = sql "select * from test_use_plan_cache20 where id=999"
-            assertTrue(result6.isEmpty())
-        }),
-        extraThread("test_truncate_partition", {
-            sql "drop table if exists test_use_plan_cache21"
-            sql """create table test_use_plan_cache21 (
-                        id int,
-                        dt int
-                       )
-                       partition by range(dt)
-                       (
-                        partition dt1 values [('1'), ('2')),
-                        partition dt2 values [('2'), ('3'))
-                       )
-                       distributed by hash(id)
-                       properties('replication_num'='1')"""
-
-            sql "insert into test_use_plan_cache21 values('2', '2')"
-            sleep(100)
-            sql "insert into test_use_plan_cache21 values('1', '1')"
-
-            // after partition changed 10s, the sql cache can be used
-            sleep(10000)
-
-            sql "set enable_nereids_planner=true"
-            sql "set enable_fallback_to_original_planner=false"
-            sql "set enable_sql_cache=true"
-
-            assertNoCache "select * from test_use_plan_cache21"
-            def result1 = sql "select * from test_use_plan_cache21"
-            assertTrue(result1.size() == 2)
-            assertHasCache "select * from test_use_plan_cache21"
-
-            sql "truncate table test_use_plan_cache21 partition dt2"
-            assertNoCache "select * from test_use_plan_cache21"
-            def result2 = sql "select * from test_use_plan_cache21"
-            assertTrue(result2.size() == 1)
-        }),
-        extraThread("remove_comment", {
-            createTestTable "test_use_plan_cache22"
-
-            // after partition changed 10s, the sql cache can be used
-            sleep(10000)
-
-            sql "set enable_nereids_planner=true"
-            sql "set enable_fallback_to_original_planner=false"
-            sql "set enable_sql_cache=true"
-
-            assertNoCache "select /*+SET_VAR(disable_nereids_rules='')*/ /*comment2*/ * from test_use_plan_cache22 order by 1, 2"
-            sql "select /*+SET_VAR(disable_nereids_rules='')*/ /*comment1*/ * from test_use_plan_cache22 order by 1, 2"
-
-            assertHasCache "select /*+SET_VAR(disable_nereids_rules='')*/ /*comment2*/ * from test_use_plan_cache22 order by 1, 2"
-        }),
-        extraThread("is_cache_profile", {
-            createTestTable "test_use_plan_cache23"
-
-            // after partition changed 10s, the sql cache can be used
-            sleep(10000)
-
-            sql "set enable_nereids_planner=true"
-            sql "set enable_fallback_to_original_planner=false"
-            sql "set enable_sql_cache=true"
-
-            int randomInt = Math.random() * 2000000000
-            sql "select ${randomInt} from test_use_plan_cache23"
-            profile("sql_cache_23_${randomInt}") {
-                run {
-                    sql "/* sql_cache_23_${randomInt} */ select ${randomInt} from test_use_plan_cache23"
-                }
+                int randomInt = (int) (Math.random() * 2000000000)
+
+                assertNoCache "select * from (select $randomInt as id)a"
+                def result1 = sql "select * from (select $randomInt as id)a"
+                assertTrue(result1.size() == 1)
+
+                assertHasCache "select * from (select $randomInt as id)a"
+                def result2 = sql "select * from (select $randomInt as id)a"
+                assertTrue(result2.size() == 1)
+
+                sql "select * from test_use_plan_cache20 limit 0"
+                assertHasCache "select * from test_use_plan_cache20 limit 0"
+                def result4 = sql "select * from test_use_plan_cache20 limit 0"
+                assertTrue(result4.isEmpty())
+
+                assertNoCache "select * from test_use_plan_cache20 where id=999"
+                def result5 = sql "select * from test_use_plan_cache20 where id=999"
+                assertTrue(result5.isEmpty())
+                assertHasCache "select * from test_use_plan_cache20 where id=999"
+                def result6 = sql "select * from test_use_plan_cache20 where id=999"
+                assertTrue(result6.isEmpty())
+            }),
+            extraThread("test_truncate_partition", {
+                sql "drop table if exists test_use_plan_cache21"
+                sql """create table test_use_plan_cache21 (
+                    id int,
+                    dt int
+                   )
+                   partition by range(dt)
+                   (
+                    partition dt1 values [('1'), ('2')),
+                    partition dt2 values [('2'), ('3'))
+                   )
+                   distributed by hash(id)
+                   properties('replication_num'='1')"""
+
+                sql "insert into test_use_plan_cache21 values('2', '2')"
+                sleep(100)
+                sql "insert into test_use_plan_cache21 values('1', '1')"
 
-                check { profileString, exception ->
-                    log.info(profileString)
-                    assertTrue(profileString.contains("Is  Cached:  Yes"))
-                }
-            }
+                // after partition changed 10s, the sql cache can be used
+                sleep(10000)
+
+                sql "set enable_nereids_planner=true"
+                sql "set enable_fallback_to_original_planner=false"
+                sql "set enable_sql_cache=true"
 
-            randomInt = Math.random() * 2000000000
-            sql "select * from (select $randomInt as id)a"
-            profile("sql_cache_23_${randomInt}_2") {
-                run {
-                    sql "/* sql_cache_23_${randomInt}_2 */ select * from (select $randomInt as id)a"
+                assertNoCache "select * from test_use_plan_cache21"
+                def result1 = sql "select * from test_use_plan_cache21"
+                assertTrue(result1.size() == 2)
+                assertHasCache "select * from test_use_plan_cache21"
+
+                sql "truncate table test_use_plan_cache21 partition dt2"
+                assertNoCache "select * from test_use_plan_cache21"
+                def result2 = sql "select * from test_use_plan_cache21"
+                assertTrue(result2.size() == 1)
+            }),
+            extraThread("remove_comment", {
+                createTestTable "test_use_plan_cache22"
+
+                // after partition changed 10s, the sql cache can be used
+                sleep(10000)
+
+                sql "set enable_nereids_planner=true"
+                sql "set enable_fallback_to_original_planner=false"
+                sql "set enable_sql_cache=true"
+
+                assertNoCache "select /*+SET_VAR(disable_nereids_rules='')*/ /*comment2*/ * from test_use_plan_cache22 order by 1, 2"
+                sql "select /*+SET_VAR(disable_nereids_rules='')*/ /*comment1*/ * from test_use_plan_cache22 order by 1, 2"
+
+                assertHasCache "select /*+SET_VAR(disable_nereids_rules='')*/ /*comment2*/ * from test_use_plan_cache22 order by 1, 2"
+            }),
+            extraThread("is_cache_profile", {
+                createTestTable "test_use_plan_cache23"
+
+                // after partition changed 10s, the sql cache can be used
+                sleep(10000)
+
+                sql "set enable_nereids_planner=true"
+                sql "set enable_fallback_to_original_planner=false"
+                sql "set enable_sql_cache=true"
+
+                int randomInt = Math.random() * 2000000000
+                sql "select ${randomInt} from test_use_plan_cache23"
+                profile("sql_cache_23_${randomInt}") {
+                    run {
+                        sql "/* sql_cache_23_${randomInt} */ select ${randomInt} from test_use_plan_cache23"
+                    }
+
+                    check { profileString, exception ->
+                        log.info(profileString)
+                        assertTrue(profileString.contains("Is  Cached:  Yes"))
+                    }
                 }
 
-                check { profileString, exception ->
-                    log.info(profileString)
-                    assertTrue(profileString.contains("Is  Cached:  Yes"))
+                randomInt = Math.random() * 2000000000
+                sql "select * from (select $randomInt as id)a"
+                profile("sql_cache_23_${randomInt}_2") {
+                    run {
+                        sql "/* sql_cache_23_${randomInt}_2 */ select * from (select $randomInt as id)a"
+                    }
+
+                    check { profileString, exception ->
+                        log.info(profileString)
+                        assertTrue(profileString.contains("Is  Cached:  Yes"))
+                    }
+                }
+            }),
+            extraThread("sql_cache_with_date_format", {
+                sql "set enable_sql_cache=true"
+                for (def i in 0..3) {
+                    def result = sql "select FROM_UNIXTIME(UNIX_TIMESTAMP(), 'yyyy-MM-dd HH:mm:ss')"
+                    assertNotEquals("yyyy-MM-dd HH:mm:ss", result[0][0])
                 }
-            }
-        }),
-        extraThread("sql_cache_with_date_format", {
-            sql "set enable_sql_cache=true"
-            for (def i in 0..3) {
-                def result = sql "select FROM_UNIXTIME(UNIX_TIMESTAMP(), 'yyyy-MM-dd HH:mm:ss')"
-                assertNotEquals("yyyy-MM-dd HH:mm:ss", result[0][0])
-            }
-        }),
-        extraThread("test_same_sql_with_different_db", {
-            def dbName1 = "test_db1"
-            def dbName2 = "test_db2"
-            def tableName = "test_cache_table"
-
-            sql "CREATE DATABASE IF NOT EXISTS ${dbName1}"
-            sql "DROP TABLE IF EXISTS ${dbName1}.${tableName}"
-            sql """
-                CREATE TABLE IF NOT EXISTS ${dbName1}.${tableName} (
-                  `k1` date NOT NULL COMMENT "",
-                  `k2` int(11) NOT NULL COMMENT ""
-                ) ENGINE=OLAP
-                DUPLICATE KEY(`k1`, `k2`)
-                COMMENT "OLAP"
-                PARTITION BY RANGE(`k1`)
-                (PARTITION p202411 VALUES [('2024-11-01'), ('2024-12-01')))
-                DISTRIBUTED BY HASH(`k1`, `k2`) BUCKETS 1
-                PROPERTIES (
-                "replication_allocation" = "tag.location.default: 1",
-                "in_memory" = "false",
-                "storage_format" = "V2"
-                )
-            """
-            sql "CREATE DATABASE IF NOT EXISTS ${dbName2}"
-            sql "DROP TABLE IF EXISTS ${dbName2}.${tableName}"
-            sql """
-                CREATE TABLE IF NOT EXISTS ${dbName2}.${tableName} (
-                  `k1` date NOT NULL COMMENT "",
-                  `k2` int(11) NOT NULL COMMENT ""
-                ) ENGINE=OLAP
-                DUPLICATE KEY(`k1`, `k2`)
-                COMMENT "OLAP"
-                PARTITION BY RANGE(`k1`)
-                (PARTITION p202411 VALUES [('2024-11-01'), ('2024-12-01')))
-                DISTRIBUTED BY HASH(`k1`, `k2`) BUCKETS 1
-                PROPERTIES (
-                "replication_allocation" = "tag.location.default: 1",
-                "in_memory" = "false",
-                "storage_format" = "V2"
-                )
-            """
-
-            sql """
-                INSERT INTO ${dbName1}.${tableName} VALUES 
-                        ("2024-11-29",0),
-                        ("2024-11-30",0)
-            """
-            // after partition changed 10s, the sql cache can be used
-            sleep(10000)
-            sql """
-                INSERT INTO ${dbName2}.${tableName} VALUES 
-                        ("2024-11-29",0)
-            """
-            // after partition changed 10s, the sql cache can be used
-            sleep(10000)
-
-            sql "set enable_sql_cache=true"
-            sql "use ${dbName1}"
-            List<List<Object>> result1 = sql """
-                SELECT COUNT(*) FROM ${tableName}
-            """
-            assertEquals(result1[0][0],2)
-
-            sql "use ${dbName2}"
-            List<List<Object>> result2 = sql """
-                SELECT COUNT(*) FROM ${tableName}
-            """
-            assertEquals(result2[0][0],1)
-
-            sql "DROP TABLE IF EXISTS ${dbName1}.${tableName}"
-            sql "DROP TABLE IF EXISTS ${dbName2}.${tableName}"
-            sql "DROP DATABASE IF EXISTS ${dbName1}"
-            sql "DROP DATABASE IF EXISTS ${dbName2}"
-        })
-    ).get()
+            }),
+            extraThread("test_same_sql_with_different_db", {
+                def dbName1 = "test_db1"
+                def dbName2 = "test_db2"
+                def tableName = "test_cache_table"
+
+                sql "CREATE DATABASE IF NOT EXISTS ${dbName1}"
+                sql "DROP TABLE IF EXISTS ${dbName1}.${tableName}"
+                sql """
+                    CREATE TABLE IF NOT EXISTS ${dbName1}.${tableName} (
+                      `k1` date NOT NULL COMMENT "",
+                      `k2` int(11) NOT NULL COMMENT ""
+                    ) ENGINE=OLAP
+                    DUPLICATE KEY(`k1`, `k2`)
+                    COMMENT "OLAP"
+                    PARTITION BY RANGE(`k1`)
+                    (PARTITION p202411 VALUES [('2024-11-01'), ('2024-12-01')))
+                    DISTRIBUTED BY HASH(`k1`, `k2`) BUCKETS 1
+                    PROPERTIES (
+                    "replication_allocation" = "tag.location.default: 1",
+                    "in_memory" = "false",
+                    "storage_format" = "V2"
+                    )
+                """
+                sql "CREATE DATABASE IF NOT EXISTS ${dbName2}"
+                sql "DROP TABLE IF EXISTS ${dbName2}.${tableName}"
+                sql """
+                    CREATE TABLE IF NOT EXISTS ${dbName2}.${tableName} (
+                      `k1` date NOT NULL COMMENT "",
+                      `k2` int(11) NOT NULL COMMENT ""
+                    ) ENGINE=OLAP
+                    DUPLICATE KEY(`k1`, `k2`)
+                    COMMENT "OLAP"
+                    PARTITION BY RANGE(`k1`)
+                    (PARTITION p202411 VALUES [('2024-11-01'), ('2024-12-01')))
+                    DISTRIBUTED BY HASH(`k1`, `k2`) BUCKETS 1
+                    PROPERTIES (
+                    "replication_allocation" = "tag.location.default: 1",
+                    "in_memory" = "false",
+                    "storage_format" = "V2"
+                    )
+                """
+
+                sql """
+                    INSERT INTO ${dbName1}.${tableName} VALUES
+                            ("2024-11-29",0),
+                            ("2024-11-30",0)
+                """
+                // after partition changed 10s, the sql cache can be used
+                sleep(10000)
+                sql """
+                    INSERT INTO ${dbName2}.${tableName} VALUES
+                            ("2024-11-29",0)
+                """
+                // after partition changed 10s, the sql cache can be used
+                sleep(10000)
+
+                sql "set enable_sql_cache=true"
+                sql "use ${dbName1}"
+                List<List<Object>> result1 = sql """
+                    SELECT COUNT(*) FROM ${tableName}
+                """
+                assertEquals(result1[0][0],2)
+
+                sql "use ${dbName2}"
+                List<List<Object>> result2 = sql """
+                    SELECT COUNT(*) FROM ${tableName}
+                """
+                assertEquals(result2[0][0],1)
+
+                sql "DROP TABLE IF EXISTS ${dbName1}.${tableName}"
+                sql "DROP TABLE IF EXISTS ${dbName2}.${tableName}"
+                sql "DROP DATABASE IF EXISTS ${dbName1}"
+                sql "DROP DATABASE IF EXISTS ${dbName2}"
+            })
+        ).get()
+    }
 }
diff --git a/regression-test/suites/nereids_syntax_p0/distribute/colocate_union_numbers.groovy b/regression-test/suites/nereids_syntax_p0/distribute/colocate_union_numbers.groovy
index b247c5bde25a6c2..3119b2ddb2c5ddc 100644
--- a/regression-test/suites/nereids_syntax_p0/distribute/colocate_union_numbers.groovy
+++ b/regression-test/suites/nereids_syntax_p0/distribute/colocate_union_numbers.groovy
@@ -25,23 +25,25 @@ suite("colocate_union_numbers") {
         """
 
     def extractFragment = { String sqlStr, String containsString, Closure<Integer> checkExchangeNum ->
-        explain {
-            sql sqlStr
-            check { result ->
-                log.info("Explain result:\n${result}")
+        retry(120, 1000) {
+            explain {
+                sql sqlStr
+                check { result ->
+                    log.info("Explain result:\n${result}")
 
-                assertTrue(result.contains(containsString))
+                    assertTrue(result.contains(containsString))
 
-                def fragmentContainsJoin = result.split("PLAN FRAGMENT")
-                        .toList()
-                        .stream()
-                        .filter { it.contains(containsString) }
-                        .findFirst()
-                        .get()
+                    def fragmentContainsJoin = result.split("PLAN FRAGMENT")
+                            .toList()
+                            .stream()
+                            .filter { it.contains(containsString) }
+                            .findFirst()
+                            .get()
 
-                log.info("Fragment:\n${fragmentContainsJoin}")
+                    log.info("Fragment:\n${fragmentContainsJoin}")
 
-                checkExchangeNum(fragmentContainsJoin.count("VEXCHANGE"))
+                    checkExchangeNum(fragmentContainsJoin.count("VEXCHANGE"))
+                }
             }
         }
     }
diff --git a/regression-test/suites/nereids_syntax_p0/distribute/prune_bucket_with_bucket_shuffle_join.groovy b/regression-test/suites/nereids_syntax_p0/distribute/prune_bucket_with_bucket_shuffle_join.groovy
index 3ce412a0075b2a1..7006b8c226e1707 100644
--- a/regression-test/suites/nereids_syntax_p0/distribute/prune_bucket_with_bucket_shuffle_join.groovy
+++ b/regression-test/suites/nereids_syntax_p0/distribute/prune_bucket_with_bucket_shuffle_join.groovy
@@ -41,23 +41,25 @@ suite("prune_bucket_with_bucket_shuffle_join") {
         """
 
     def extractFragment = { String sqlStr, String containsString, Closure<Integer> checkExchangeNum ->
-        explain {
-            sql sqlStr
-            check { result ->
-                log.info("Explain result:\n${result}")
+        retry(120, 1000) {
+            explain {
+                sql sqlStr
+                check { result ->
+                    log.info("Explain result:\n${result}")
 
-                assertTrue(result.contains(containsString))
+                    assertTrue(result.contains(containsString))
 
-                def fragmentContainsJoin = result.split("PLAN FRAGMENT")
-                        .toList()
-                        .stream()
-                        .filter { it.contains(containsString) }
-                        .findFirst()
-                        .get()
+                    def fragmentContainsJoin = result.split("PLAN FRAGMENT")
+                            .toList()
+                            .stream()
+                            .filter { it.contains(containsString) }
+                            .findFirst()
+                            .get()
 
-                log.info("Fragment:\n${fragmentContainsJoin}")
+                    log.info("Fragment:\n${fragmentContainsJoin}")
 
-                checkExchangeNum(fragmentContainsJoin.count("VEXCHANGE"))
+                    checkExchangeNum(fragmentContainsJoin.count("VEXCHANGE"))
+                }
             }
         }
     }
diff --git a/regression-test/suites/nereids_syntax_p0/explain.groovy b/regression-test/suites/nereids_syntax_p0/explain.groovy
index fb6af28dd443f4c..5b1bfa1f8ae8972 100644
--- a/regression-test/suites/nereids_syntax_p0/explain.groovy
+++ b/regression-test/suites/nereids_syntax_p0/explain.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("nereids_explain") {
+suite("explain") {
     sql """
         SET enable_nereids_planner=true
     """
@@ -29,7 +29,6 @@ suite("nereids_explain") {
         contains "sum(2) + sum(lo_suppkey)"
     }
 
-
     explain {
         sql("physical plan select 100")
         contains "PhysicalOneRowRelation"
@@ -42,7 +41,7 @@ suite("nereids_explain") {
 
     explain {
         sql("parsed plan select 100")
-        contains "UnboundOneRowRelation"
+        contains "LogicalOneRowRelation"
     }
 
     explain {

From 9e7e33cb05dacff08e78754988203e7a8bc19815 Mon Sep 17 00:00:00 2001
From: HappenLee <happenlee@selectdb.com>
Date: Tue, 7 Jan 2025 10:46:41 +0800
Subject: [PATCH 011/140] [opt](vec) opt the date/datetime is invalid func
 performance (#41986)

## Proposed changes

before : cost 40s
```
select count(month(d1)) from tb1;
```


after : cost 28s
```
select count(month(d1)) from tb1;
```
---
 be/src/util/time_lut.cpp                    |  4 ----
 be/src/util/time_lut.h                      |  4 +++-
 be/src/vec/functions/date_time_transforms.h |  3 +++
 be/src/vec/runtime/vdatetime_value.cpp      | 22 ---------------------
 be/src/vec/runtime/vdatetime_value.h        | 14 ++++++++++++-
 5 files changed, 19 insertions(+), 28 deletions(-)

diff --git a/be/src/util/time_lut.cpp b/be/src/util/time_lut.cpp
index 616541d411f6364..cb2aecb72897288 100644
--- a/be/src/util/time_lut.cpp
+++ b/be/src/util/time_lut.cpp
@@ -87,10 +87,6 @@ uint32_t calc_days_in_year(uint32_t year) {
     return is_leap(year) ? 366 : 365;
 }
 
-bool is_leap(uint32_t year) {
-    return ((year % 4) == 0) && ((year % 100 != 0) || ((year % 400) == 0 && year));
-}
-
 uint8_t calc_weekday(uint64_t day_nr, bool is_sunday_first_day) {
     return (day_nr + 5L + (is_sunday_first_day ? 1L : 0L)) % 7;
 }
diff --git a/be/src/util/time_lut.h b/be/src/util/time_lut.h
index 3955d8c8ed79dcd..f98560c45e1a434 100644
--- a/be/src/util/time_lut.h
+++ b/be/src/util/time_lut.h
@@ -36,7 +36,9 @@ uint32_t calc_daynr(uint16_t year, uint8_t month, uint8_t day);
 
 uint8_t calc_weekday(uint64_t day_nr, bool is_sunday_first_day);
 
-bool is_leap(uint32_t year);
+inline bool is_leap(uint32_t year) {
+    return ((year % 4) == 0) && ((year % 100 != 0) || ((year % 400) == 0 && year));
+}
 
 uint32_t calc_days_in_year(uint32_t year);
 
diff --git a/be/src/vec/functions/date_time_transforms.h b/be/src/vec/functions/date_time_transforms.h
index b65840445ca4fb7..301effe80e0df82 100644
--- a/be/src/vec/functions/date_time_transforms.h
+++ b/be/src/vec/functions/date_time_transforms.h
@@ -353,6 +353,9 @@ struct Transformer {
             auto res = Transform::execute(vec_from[i]);
             using RESULT_TYPE = std::decay_t<decltype(res)>;
             vec_to[i] = cast_set<ToType, RESULT_TYPE, false>(res);
+        }
+
+        for (size_t i = 0; i < size; ++i) {
             null_map[i] = !((typename DateTraits<typename Transform::OpArgType>::T&)(vec_from[i]))
                                    .is_valid_date();
         }
diff --git a/be/src/vec/runtime/vdatetime_value.cpp b/be/src/vec/runtime/vdatetime_value.cpp
index 026648319d4be4c..8e053c93447602c 100644
--- a/be/src/vec/runtime/vdatetime_value.cpp
+++ b/be/src/vec/runtime/vdatetime_value.cpp
@@ -1936,28 +1936,6 @@ std::size_t hash_value(VecDateTimeValue const& value) {
     return HashUtil::hash(&value, sizeof(VecDateTimeValue), 0);
 }
 
-template <typename T>
-bool DateV2Value<T>::is_invalid(uint32_t year, uint32_t month, uint32_t day, uint8_t hour,
-                                uint8_t minute, uint8_t second, uint32_t microsecond,
-                                bool only_time_part) {
-    if (hour >= 24 || minute >= 60 || second >= 60 || microsecond > 999999) {
-        return true;
-    }
-    if (only_time_part) {
-        return false;
-    }
-    if (year > MAX_YEAR) {
-        return true;
-    }
-    if (month == 2 && day == 29 && doris::is_leap(year)) {
-        return false;
-    }
-    if (month == 0 || month > 12 || day > S_DAYS_IN_MONTH[month] || day == 0) {
-        return true;
-    }
-    return false;
-}
-
 template <typename T>
 void DateV2Value<T>::format_datetime(uint32_t* date_val, bool* carry_bits) const {
     // ms
diff --git a/be/src/vec/runtime/vdatetime_value.h b/be/src/vec/runtime/vdatetime_value.h
index cfe9a368e83d4bc..f724f918828c77b 100644
--- a/be/src/vec/runtime/vdatetime_value.h
+++ b/be/src/vec/runtime/vdatetime_value.h
@@ -872,7 +872,19 @@ class DateV2Value {
     // Return true if range or date is invalid
     static bool is_invalid(uint32_t year, uint32_t month, uint32_t day, uint8_t hour,
                            uint8_t minute, uint8_t second, uint32_t microsecond,
-                           bool only_time_part = false);
+                           bool only_time_part = false) {
+        if constexpr (is_datetime) {
+            if (hour >= 24 || minute >= 60 || second >= 60 || microsecond > 999999) {
+                return true;
+            }
+            if (only_time_part) {
+                return false;
+            }
+        }
+        return year > MAX_YEAR || !day || !month || month > 12 ||
+               (day > 28 && ((month != 2 && day > S_DAYS_IN_MONTH[month]) ||
+                             (month == 2 && day > 28 + doris::is_leap(year))));
+    }
 
     [[nodiscard]] bool check_range_and_set_time(uint16_t year, uint8_t month, uint8_t day,
                                                 uint8_t hour, uint8_t minute, uint8_t second,

From 438a9cd19cc602847df605584b21bc87663a1ee8 Mon Sep 17 00:00:00 2001
From: LiBinfeng <libinfeng@selectdb.com>
Date: Tue, 7 Jan 2025 11:00:18 +0800
Subject: [PATCH 012/140] [fix](Nereids) fix fe folding constant of string
 functions and add more cases (#45233)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What problem does this PR solve?
Issue Number: #44666
Related PR: #40441

Problem Summary:
- select substring_index('哈哈哈AAA','A', 1);
String.split function has second parameter 'limit', which is default
zero. When 'limit' is zero, it means it would remove trailing
empty strings split of '哈哈哈AAA', which would be '哈哈哈' only. But what we
expect is '哈哈哈', '','','' when part function is used by substring index.
So we should change splitpart limit to -1 to enable trailing empty
character in splitpart list
- reorganize fold constant of string functions in fe and add more cases
---
 .../executable/StringArithmetic.java          |   52 +-
 .../fold_constant_string_arithmatic.groovy    | 1367 ++++++++---------
 2 files changed, 677 insertions(+), 742 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java
index 7b37ca51c53fa48..bc056a03bcb40ba 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java
@@ -318,7 +318,20 @@ public static Expression right(StringLikeLiteral first, IntegerLiteral second) {
      */
     @ExecFunction(name = "locate")
     public static Expression locate(StringLikeLiteral first, StringLikeLiteral second) {
-        return new IntegerLiteral(second.getValue().trim().indexOf(first.getValue()) + 1);
+        return new IntegerLiteral(second.getValue().indexOf(first.getValue()) + 1);
+    }
+
+    /**
+     * Executable arithmetic functions Locate
+     */
+    @ExecFunction(name = "locate")
+    public static Expression locate(StringLikeLiteral first, StringLikeLiteral second, IntegerLiteral third) {
+        int result = second.getValue().indexOf(first.getValue()) + 1;
+        if (third.getValue() <= 0 || !substringImpl(second.getValue(), third.getValue(),
+                second.getValue().length()).contains(first.getValue())) {
+            result = 0;
+        }
+        return new IntegerLiteral(result);
     }
 
     /**
@@ -333,12 +346,14 @@ public static Expression instr(StringLikeLiteral first, StringLikeLiteral second
      * Executable arithmetic functions Ascii
      */
     @ExecFunction(name = "ascii")
-    public static Expression ascii(StringLikeLiteral first) {
+    public static Expression ascii(StringLikeLiteral first) throws UnsupportedEncodingException {
         if (first.getValue().length() == 0) {
             return new IntegerLiteral(0);
         }
-        char firstChar = first.getValue().charAt(0);
-        return new IntegerLiteral(firstChar);
+        String character = first.getValue();
+        byte[] utf8Bytes = character.getBytes("UTF-8");
+        int firstByteAscii = utf8Bytes[0] & 0xFF;
+        return new IntegerLiteral(firstByteAscii);
     }
 
     /**
@@ -583,7 +598,7 @@ public static Expression fieldVarchar(StringLikeLiteral first, VarcharLiteral...
     }
 
     private static int findStringInSet(String target, String input) {
-        String[] split = input.split(",");
+        String[] split = input.split(",", -1);
         for (int i = 0; i < split.length; i++) {
             if (split[i].equals(target)) {
                 return i + 1;
@@ -605,6 +620,10 @@ public static Expression findInSetVarchar(StringLikeLiteral first, StringLikeLit
      */
     @ExecFunction(name = "repeat")
     public static Expression repeat(StringLikeLiteral first, IntegerLiteral second) {
+        // when it is too large for fe to make result string, do not folding on fe, limit 1 MB
+        if ((first.getValue().length() * second.getValue()) > 1000000) {
+            throw new AnalysisException("repeat too large to fold const by fe");
+        }
         StringBuilder sb = new StringBuilder();
         for (int i = 0; i < second.getValue(); i++) {
             sb.append(first.getValue());
@@ -627,6 +646,10 @@ public static Expression reverseVarchar(StringLikeLiteral first) {
      */
     @ExecFunction(name = "space")
     public static Expression space(IntegerLiteral first) {
+        // when it is too large for fe to make result string, do not folding on fe, limit 1 MB
+        if (first.getValue() > 1000000) {
+            throw new AnalysisException("space too large to fold const by fe");
+        }
         StringBuilder sb = new StringBuilder();
         for (int i = 0; i < first.getValue(); i++) {
             sb.append(' ');
@@ -639,7 +662,7 @@ public static Expression space(IntegerLiteral first) {
      */
     @ExecFunction(name = "split_by_char")
     public static Expression splitByChar(StringLikeLiteral first, StringLikeLiteral second) {
-        String[] result = first.getValue().split(second.getValue());
+        String[] result = first.getValue().split(second.getValue(), -1);
         List<Literal> items = new ArrayList<>();
         for (int i = 1; i < result.length; i++) {
             items.add((Literal) castStringLikeLiteral(first, result[i]));
@@ -666,16 +689,16 @@ public static Expression splitPart(StringLikeLiteral first, StringLikeLiteral ch
             if (".$|()[{^?*+\\".contains(separator) || separator.startsWith("\\")) {
                 separator = "\\" + separator;
             }
-            parts = sb.reverse().toString().split(separator);
+            parts = sb.reverse().toString().split(separator, -1);
         } else {
             if (".$|()[{^?*+\\".contains(separator) || separator.startsWith("\\")) {
                 separator = "\\" + separator;
             }
-            parts = first.getValue().split(separator);
+            parts = first.getValue().split(separator, -1);
         }
 
         if (parts.length < Math.abs(number.getValue()) || number.getValue() == 0) {
-            if (parts.length == Math.abs(number.getValue()) - 1) {
+            if (parts.length == Math.abs(number.getValue())) {
                 if (number.getValue() < 0 && first.getValue().startsWith(chr.getValue())
                         || number.getValue() > 0 && first.getValue().endsWith(chr.getValue())) {
                     return castStringLikeLiteral(first, "");
@@ -695,7 +718,10 @@ public static Expression splitPart(StringLikeLiteral first, StringLikeLiteral ch
      */
     @ExecFunction(name = "substring_index")
     public static Expression substringIndex(StringLikeLiteral first, StringLikeLiteral chr, IntegerLiteral number) {
-        String[] parts = first.getValue().split(chr.getValue());
+        if (chr.getValue().isEmpty()) {
+            return chr;
+        }
+        String[] parts = first.getValue().split(chr.getValue(), -1);
         if (Math.abs(number.getValue()) >= parts.length) {
             return first;
         }
@@ -907,13 +933,13 @@ public static Expression extractUrlParameter(StringLikeLiteral first, StringLike
             return castStringLikeLiteral(first, "");
         }
 
-        String[] urlParts = first.getValue().split("\\?");
+        String[] urlParts = first.getValue().split("\\?", -1);
         if (urlParts.length > 1) {
             String query = urlParts[1];
-            String[] pairs = query.split("&");
+            String[] pairs = query.split("&", -1);
 
             for (String pair : pairs) {
-                String[] keyValue = pair.split("=");
+                String[] keyValue = pair.split("=", -1);
                 if (second.getValue().equals(keyValue[0])) {
                     return castStringLikeLiteral(first, keyValue[1]);
                 }
diff --git a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_string_arithmatic.groovy b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_string_arithmatic.groovy
index 545ac19576be990..e3ed0024b6f11f2 100644
--- a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_string_arithmatic.groovy
+++ b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_string_arithmatic.groovy
@@ -16,763 +16,672 @@
 // under the License.
 
 suite("fold_constant_string_arithmatic") {
-    def db = "fold_constant_string_arithmatic"
-    sql "create database if not exists ${db}"
-
     sql "set enable_nereids_planner=true"
     sql "set enable_fallback_to_original_planner=false"
     sql "set enable_fold_constant_by_be=false"
-
-    testFoldConst("SELECT  Concat('Hello', ' ', 'World')")
-    testFoldConst("SELECT  Substring('Hello World', 1, 5)")
-    testFoldConst("SELECT  Substring('1', 1, 1)")
-    testFoldConst("select  100, 'abc', substring('abc', 1, 2), substring(substring('abcdefg', 4, 3), 1, 2), null")
-    testFoldConst("SELECT  Length('Hello World')")
-    testFoldConst("SELECT  Lower('Hello World')")
-    testFoldConst("SELECT  Upper('Hello World')")
-    testFoldConst("SELECT  Trim('  Hello World  ')")
-    testFoldConst("SELECT  Trim('11111', 11)")
-    testFoldConst("SELECT  Ltrim('  Hello World  ')")
-    testFoldConst("SELECT  LTrim(' 11111', 11)")
-    testFoldConst("SELECT  LTrim('11111 ', 11)")
-    testFoldConst("SELECT  Rtrim('  Hello World  ')")
-    testFoldConst("SELECT  RTrim('11111 ', 11)")
-    testFoldConst("SELECT  RTrim(' 11111', 11)")
-    testFoldConst("SELECT  Replace('Hello World', 'World', 'Everyone')")
-    testFoldConst("SELECT  Left('Hello World', 5)")
-    testFoldConst("SELECT  Right('Hello World', 5)")
-    testFoldConst("SELECT  Locate('World', 'Hello World')")
-    testFoldConst("SELECT  Instr('Hello World', 'World')")
-    testFoldConst("SELECT  Ascii('A')")
-    testFoldConst("SELECT  Bin(5)")
-    testFoldConst("SELECT  Hex(255)")
-    testFoldConst("SELECT  Unhex('FF')")
-    testFoldConst("SELECT  Concat_Ws('-', '2024', '09', '02')")
-    testFoldConst("SELECT  Char(65)")
-    testFoldConst("SELECT  Character_Length('Hello World')")
-    testFoldConst("SELECT  Length('你')")
-    testFoldConst("SELECT  Initcap('hello world')")
-    testFoldConst("SELECT  Md5('Hello World')")
-    testFoldConst("SELECT  Md5Sum('Hello World')")
-//    testFoldConst("SELECT  JsonExtract('{"key": "value"}', '$.key')")
-//    testFoldConst("SELECT  JsonbExtractString('{"key": "value"}', '$.key')")
-//    testFoldConst("SELECT  JsonContains('{"key": "value"}', '"key"')")
-//    testFoldConst("SELECT  JsonLength('{"key1": "value1", "key2": "value2"}')")
-//    testFoldConst("SELECT  JsonObject('key', 'value')")
-//    testFoldConst("SELECT  JsonArray('value1', 'value2')")
-//    testFoldConst("SELECT  JsonKeys('{"key1": "value1", "key2": "value2"}')")
-//    testFoldConst("SELECT  JsonInsert('{"key1": "value1"}', '$.key2', 'value2')")
-//    testFoldConst("SELECT  JsonReplace('{"key1": "value1"}', '$.key1', 'new_value')")
-//    testFoldConst("SELECT  JsonSet('{"key1": "value1"}', '$.key2', 'value2')")
-//    testFoldConst("SELECT  Json_Quote('Hello World')")
-//    testFoldConst("SELECT  Json_UnQuote('"Hello World"')")
-    testFoldConst("SELECT  Field('b', 'a', 'b', 'c')")
-    testFoldConst("SELECT  Find_In_Set('b', 'a,b,c')")
-    testFoldConst("SELECT  Repeat('Hello', 3)")
-    testFoldConst("SELECT  Reverse('Hello')")
-    testFoldConst("SELECT  length(Space(10))")
-//    testFoldConst("SELECT  Split_By_Char('a,b,c',',')")  has bug in be execution
-    testFoldConst("SELECT  Split_By_String('a::b::c', '::')")
-    testFoldConst("SELECT  Split_Part('a,b,c', ',', 2)")
-    testFoldConst("SELECT  Substring_Index('a,b,c', ',', 2)")
-    testFoldConst("SELECT  Strcmp('abc', 'abd')")
-    testFoldConst("SELECT  StrLeft('Hello World', 5)")
-    testFoldConst("SELECT  StrRight('Hello World', 5)")
-    testFoldConst("SELECT  Overlay('abcdef', '123', 3, 2)")
-    testFoldConst("SELECT  Parse_Url('http://www.example.com/path?query=abc', 'HOST')")
-    testFoldConst("SELECT  Url_Decode('+Hello+World+')")
-    testFoldConst("SELECT  Url_Encode(' Hello World ')")
-
-    // Substring with negative start index
-    // Expected behavior: Depending on the SQL engine, might return an empty string or error.
-    testFoldConst("SELECT Substring('Hello World', -1, 5)")
-
-    // Substring with length exceeding the string length
-    // Expected behavior: Return 'Hello' as the length exceeds the string length.
-    testFoldConst("SELECT Substring('Hello', 1, 10)")
-
-    // Left with length greater than string length
-    // Expected behavior: Return 'Hello'.
-    testFoldConst("SELECT Left('Hello', 10)")
-
-    // Right with length greater than string length
-    // Expected behavior: Return 'Hello'.
-    testFoldConst("SELECT Right('Hello', 10)")
-
-    // SplitPart with part number greater than the number of parts
-    // Expected behavior: Return an empty string or error.
-    testFoldConst("SELECT Split_Part('a,b,c', ',', 5)")
-
-    // SplitPart with negative part number
-    // Expected behavior: Return an empty string or error.
-    testFoldConst("SELECT Split_Part('a,b,c', ',', -1)")
-
-    // Locate with the substring not present
-    // Expected behavior: Return 0 as 'World' is not found.
-    testFoldConst("SELECT Locate('World', 'Hello')")
-
-    // Instr with the substring not present
-    // Expected behavior: Return 0 as 'World' is not found.
-    testFoldConst("SELECT Instr('Hello', 'World')")
-
-    // Replace with an empty search string
-    // Expected behavior: Some SQL engines may treat this as a no-op, others might throw an error.
-    testFoldConst("SELECT Replace('Hello World', '', 'Everyone')")
-
-    // Replace with an empty replacement string
-    // Expected behavior: Return 'Hello '.
-    testFoldConst("SELECT Replace('Hello World', 'World', '')")
-
-    // Concat with NULL values
-    // Expected behavior: Depending on the SQL engine, may return 'HelloWorld' or NULL.
-    testFoldConst("SELECT Concat('Hello', NULL, 'World')")
-
-    // Ltrim with a string that has no leading spaces
-    // Expected behavior: Return 'Hello'.
-    testFoldConst("SELECT Ltrim('Hello')")
-
-    // Rtrim with a string that has no trailing spaces
-    // Expected behavior: Return 'Hello'.
-    testFoldConst("SELECT Rtrim('Hello')")
-
-    // JsonExtract with an invalid JSON path
-    // Expected behavior: Return NULL or an empty string.
-//    testFoldConst("SELECT Json_Extract('{"key": "value"}', '$.invalid')")
-
-    // JsonLength with a non-JSON string
-    // Expected behavior: Return NULL or error.
-    testFoldConst("SELECT Json_Length('Hello World')")
-
-    // Field with a value not present in the list
-    // Expected behavior: Return 0 as 'd' is not found.
-    testFoldConst("SELECT Field('d', 'a', 'b', 'c')")
-
-    // FindInSet with a value not present in the set
-    // Expected behavior: Return 0 as 'd' is not found.
-    testFoldConst("SELECT Find_In_Set('d', 'a,b,c')")
-
-    // Repeat with a negative repeat count
-    // Expected behavior: Return an empty string or error.
-    testFoldConst("SELECT Repeat('Hello', -3)")
-
-    // Space with a negative number of spaces
-    // Expected behavior: Return an empty string or error.
-    testFoldConst("SELECT Space(-5)")
-
-    // SplitByChar with a delimiter not present in the string
-    // Expected behavior: Return the original string in a single element array.
-//    testFoldConst("SELECT Split_By_Char('abc', ',')")
-
-    // SplitByString with a delimiter not present in the string
-    // Expected behavior: Return the original string in a single element array.
-    testFoldConst("SELECT Split_By_String('::', 'abc')")
-
-    // Strcmp with two identical strings
-    // Expected behavior: Return 0 as the strings are equal.
-    testFoldConst("SELECT Strcmp('abc', 'abc')")
-
-    // Strcmp with a null string
-    // Expected behavior: Return NULL or -1 depending on the SQL engine.
-    testFoldConst("SELECT Strcmp('abc', NULL)")
-
-    // Hex with a negative number
-    // Expected behavior: Return the hexadecimal representation of the two's complement, or an error depending on the SQL engine.
-    testFoldConst("SELECT Hex(-255)")
-
-    // Unhex with an invalid hexadecimal string
-    // Expected behavior: Return NULL or error as 'GHIJ' is not a valid hex string.
-    testFoldConst("SELECT Unhex('GHIJ')")
-
-    // JsonReplace with a path that does not exist
-    // Expected behavior: Depending on the engine, might return the original JSON or an error.
-//    testFoldConst("SELECT Json_Replace('{"key": "value"}', '$.nonexistent', 'new_value')")
-
-    // UrlDecode with an invalid percent-encoded string
-    // Expected behavior: Return NULL or error due to invalid encoding.
-    // testFoldConst("SELECT Url_Decode('%ZZHello%20World')")
-
-    testFoldConst("select elt(0, \"hello\", \"doris\")")
-    testFoldConst("select elt(1, \"hello\", \"doris\")")
-    testFoldConst("select elt(2, \"hello\", \"doris\")")
-    testFoldConst("select elt(3, \"hello\", \"doris\")")
-    testFoldConst("select c1, c2, elt(c1, c2) from (select number as c1, 'varchar' as c2 from numbers('number'='5') where number > 0) a")
-
+    
+    // append_trailing_char_if_absent
+    testFoldConst("select append_trailing_char_if_absent('', '!')")
+    testFoldConst("select append_trailing_char_if_absent(12345, '!')")
     testFoldConst("select append_trailing_char_if_absent('a','c')")
     testFoldConst("select append_trailing_char_if_absent('ac','c')")
+    testFoldConst("select append_trailing_char_if_absent(cast('a' as string), cast('c' as string))")
+    testFoldConst("select append_trailing_char_if_absent(cast('ac' as string), cast('c' as string))")
+    testFoldConst("select append_trailing_char_if_absent('hello!', '!')")
+    testFoldConst("select append_trailing_char_if_absent('hello', '😊')")
+    testFoldConst("select append_trailing_char_if_absent('hello😊', '😊')")
+    testFoldConst("select append_trailing_char_if_absent('hello😊', '(ಥ _ ಥ)')")
+    testFoldConst("select append_trailing_char_if_absent('hello', ' ')")
+    testFoldConst("select append_trailing_char_if_absent('hello', '')")
+    testFoldConst("select append_trailing_char_if_absent('hello', '?')")
+    testFoldConst("select append_trailing_char_if_absent('hello?', '!')")
+    testFoldConst("select append_trailing_char_if_absent('hello', '1')")
+    testFoldConst("select append_trailing_char_if_absent('hello', 1)")
+    testFoldConst("select append_trailing_char_if_absent('hello', 'ab')")
+    testFoldConst("select append_trailing_char_if_absent('hello', NULL)")
+    testFoldConst("select append_trailing_char_if_absent('hello', 'ell')")
+    testFoldConst("select append_trailing_char_if_absent('hello', 'ello')")
+    testFoldConst("select append_trailing_char_if_absent('ello', 'hello')")
     testFoldConst("select append_trailing_char_if_absent('it','a')")
-
+    testFoldConst("select append_trailing_char_if_absent(NULL, '!')")
+    testFoldConst("select append_trailing_char_if_absent('This is a very long string', '.')")
+    testFoldConst("select append_trailing_char_if_absent('Привет', '!')")
+    testFoldConst("select append_trailing_char_if_absent('Привет', 'вет')")
+    testFoldConst("select append_trailing_char_if_absent('こんにちは', '!')")
+    testFoldConst("select append_trailing_char_if_absent('\n\t', '\n')")
+    testFoldConst("select append_trailing_char_if_absent('こんにちは', 'ちは')")
+    
+    // ascii
+    testFoldConst("select ascii('!')")
     testFoldConst("select ascii('1')")
     testFoldConst("select ascii('a')")
     testFoldConst("select ascii('A')")
-    testFoldConst("select ascii('!')")
-
-    testFoldConst("select bit_length(\"abc\")")
-
-    testFoldConst("select char_length(\"abc\")")
-
-    testFoldConst("select concat(\"a\", \"b\")")
-    testFoldConst("select concat(\"a\", \"b\", \"c\")")
-    testFoldConst("select concat(\"a\", null, \"c\")")
-
-    testFoldConst("select concat_ws(\"or\", \"d\", \"is\")")
-    testFoldConst("select concat_ws(NULL, \"d\", \"is\")")
-    testFoldConst("select concat_ws(\"or\", \"d\", NULL,\"is\")")
-    testFoldConst("select concat_ws(\"or\", [\"d\", \"is\"])")
-    testFoldConst("select concat_ws(NULL, [\"d\", \"is\"])")
-    testFoldConst("select concat_ws(\"or\", [\"d\", NULL,\"is\"])")
-    testFoldConst("select concat_ws(\"or\", [\"d\", \"\",\"is\"])")
-
-    testFoldConst("select ends_with(\"Hello doris\", \"doris\")")
-    testFoldConst("select ends_with(\"Hello doris\", \"Hello\")")
-
-    testFoldConst("select find_in_set(\"b\", \"a,b,c\")")
-    testFoldConst("select find_in_set(\"d\", \"a,b,c\")")
-    testFoldConst("select find_in_set(null, \"a,b,c\")")
-    testFoldConst("select find_in_set(\"a\", null)")
+    testFoldConst("select ascii('こ')")
+    testFoldConst("select ascii('안')")
+    testFoldConst("select ascii('안こ')")
+    testFoldConst("select ascii('')")
+    testFoldConst("select ascii('中')")
+    
+    // bin
+    testFoldConst("select bin(5)")
+    testFoldConst("select bin(-5)")
+    testFoldConst("select bin(9223372036854775807)")
+    testFoldConst("select bin(9223372036854775808)")
+    testFoldConst("select bin(-9223372036854775809)")
+    
+    // bit_length
+    testFoldConst("select bit_length('abc')")
+    testFoldConst("select bit_length(cast('abc' as string))")
+    testFoldConst("select bit_length('こんにちは世界')")
+    testFoldConst("select bit_length('안녕하세요 세계!')")
+    testFoldConst("select bit_length('')")
+    
+    // char
+    testFoldConst("select char(65)")
+    testFoldConst("select char(-1)")
+    testFoldConst("select char(65535)")
+    
+    // character_length
+    testFoldConst("select character_length(cast('Hello World' as string))")
+    testFoldConst("select character_length('Hello World')")
+    testFoldConst("select character_length('你好 世界')")
+    testFoldConst("select character_length(' Hello World')")
+    testFoldConst("select character_length('  你好 世界')")
+    testFoldConst("select character_length('Hello World  ')")
+    testFoldConst("select character_length('  你好 世界  ')")
+    testFoldConst("select char_length('abc')")
+    testFoldConst("select char_length(cast('abc' as string))")
+    testFoldConst("select char_length('你好 世界')")
+    testFoldConst("select char_length('  abc')")
+    testFoldConst("select char_length('  你好 世界')")
+    testFoldConst("select char_length('你好 世界  ')")
+
+    // concat
+    testFoldConst("select concat('a', 'b')")
+    testFoldConst("select concat('a', 'b', 'c')")
+    testFoldConst("select concat('a', null, 'c')")
+    testFoldConst("select concat('Hello', NULL, 'World')")
+    testFoldConst("select concat('Hello', ' ', 'World')")
+    testFoldConst("select concat('你好', ' ', '世界')")
+    testFoldConst("select concat('', '你好', ' ', '世界')")
+    testFoldConst("select concat('你好', ' ', '世界', '')")
+    
+    // concat_ws
+    testFoldConst("select concat_ws('-', '2024', '09', '02')")
+    testFoldConst("select concat_ws('', '2024', '09', '02')")
+    testFoldConst("select concat_ws('-', '', '2024', '09', '02')")
+    testFoldConst("select concat_ws(NULL, ['d', 'is'])")
+    testFoldConst("select concat_ws(NULL, 'd', 'is')")
+    testFoldConst("select concat_ws('or', ['d', '','is'])")
+    testFoldConst("select concat_ws('or', ['d', 'is'])")
+    testFoldConst("select concat_ws('or', 'd', 'is')")
+    testFoldConst("select concat_ws('or', ['d', NULL,'is'])")
+    testFoldConst("select concat_ws('or', 'd', NULL,'is')")
+    testFoldConst("select concat_ws(' ', '你好', '世界')")
+    testFoldConst("select concat_ws(' ', [])")
+    
+    // elt
+    testFoldConst("select elt(0, cast('hello' as string), cast('doris' as string))")
+    testFoldConst("select elt(0, 'hello', 'doris')")
+    testFoldConst("select elt(1, cast('hello' as string), cast('doris' as string))")
+    testFoldConst("select elt(1, 'hello', 'doris')")
+    testFoldConst("select elt(2, cast('hello' as string), cast('doris' as string))")
+    testFoldConst("select elt(2, 'hello', 'doris')")
+    testFoldConst("select elt(3, cast('hello' as string), cast('doris' as string))")
+    testFoldConst("select elt(3, 'hello', 'doris')")
+    testFoldConst("select c1, c2, elt(c1, c2) from (select number as c1, 'varchar' as c2 from numbers('number'='5') where number > 0) a")
 
+    // ends_with
+    testFoldConst("select ends_with(cast('Hello doris' as string), cast('doris' as string))")
+    testFoldConst("select ends_with('Hello doris', 'doris')")
+    testFoldConst("select ends_with('こんにちは世界！안녕하세요 세계', '안녕하세요 세계')")
+    testFoldConst("select ends_with('안녕하세요 세계こんにちは世界！', 'こんにちは世界！')")
+    testFoldConst("select ends_with('안녕하세요 세계こんにちは世界', 'こんにちは世界')")
+    testFoldConst("select ends_with('안녕하세요 세계こんにちは世界', 'こんにちは')")
+    testFoldConst("select ends_with('Hello doris', '')")
+    testFoldConst("select ends_with('', 'Hello doris')")
+    testFoldConst("select ends_with(null, 'Hello doris')")
+    testFoldConst("select ends_with('Hello doris', null)")
+    testFoldConst("select ends_with(' ', '')")
+    testFoldConst("select ends_with(' ', ' ')")
+    testFoldConst("select ends_with('', ' ')")
+    testFoldConst("select ends_with('', '')")
+
+    // field
+    testFoldConst("select field('b', 'a', 'b', 'c')")
+    testFoldConst("select field('d', 'a', 'b', 'c')")
+    testFoldConst("select field('こ', 'ん', 'に', 'ち', 'こ')")
+    testFoldConst("select field('=', '+', '=', '=', 'こ')")
+    testFoldConst("select field('==', '+', '=', '==', 'こ')")
+    testFoldConst("select field('=', '+', '==', '==', 'こ')")
+    
+    // find_in_set
+    testFoldConst("select find_in_set('a', null)")
+    testFoldConst("select find_in_set('b', 'a,b,c')")
+    testFoldConst("select find_in_set('b', ' a,b,c')")
+    testFoldConst("select find_in_set('b', 'a ,b,c')")
+    testFoldConst("select find_in_set('b', 'a, b,c')")
+    testFoldConst("select find_in_set('b', 'a,b,c ')")
+    testFoldConst("select find_in_set('a,b,c ', 'a,b,c')")
+    testFoldConst("select find_in_set('b', 'a,b,c')")
+    testFoldConst("select find_in_set(cast('a' as string), NULL)")
+    testFoldConst("select find_in_set(cast('b' as string), cast('a,b,c' as string))")
+    testFoldConst("select find_in_set(cast('b' as string), cast('a,b,c' as string))")
+    testFoldConst("select find_in_set(cast('d' as string), cast('a,b,c' as string))")
+    testFoldConst("select find_in_set(cast('d' as string), cast('a,b,c' as string))")
+    testFoldConst("select find_in_set('d', 'a,b,c')")
+    testFoldConst("select find_in_set('d', 'a,b,c')")
+    testFoldConst("select find_in_set(null, 'a,b,c')")
+    testFoldConst("select find_in_set(NULL, cast('a,b,c' as string))")
+    testFoldConst("SELECT find_in_set('A', '哈哈哈AAA')")
+    testFoldConst("SELECT find_in_set('哈','哈哈哈AAA')")
+    testFoldConst("SELECT find_in_set(' ','哈哈哈AAA')")
+    testFoldConst("SELECT find_in_set('','哈哈哈AAA')")
+    testFoldConst("SELECT find_in_set(',','a,')")
+    testFoldConst("SELECT find_in_set(',','哈哈哈AAA')")
+    
+    // hex
+    testFoldConst("select hex('@')")
     testFoldConst("select hex('1')")
+    testFoldConst("select hex(-1)")
+    testFoldConst("select hex(-1)")
     testFoldConst("select hex('12')")
-    testFoldConst("select hex('@')")
-    testFoldConst("select hex('A')")
     testFoldConst("select hex(12)")
-    testFoldConst("select hex(-1)")
+    testFoldConst("select hex(12)")
+    testFoldConst("select hex(-255)")
+    testFoldConst("select hex(-255)")
+    testFoldConst("select hex(255)")
+    testFoldConst("select hex(255)")
+    testFoldConst("select hex('A')")
+    testFoldConst("select hex(cast('12' as string))")
+    testFoldConst("select hex(cast('1' as string))")
+    testFoldConst("select hex(cast('A' as string))")
+    testFoldConst("select hex(cast('@' as string))")
+    testFoldConst("select hex(cast('hello,doris' as string))")
     testFoldConst("select hex('hello,doris')")
-
-    testFoldConst("select unhex('@')")
-    testFoldConst("select unhex('68656C6C6F2C646F726973')")
-    testFoldConst("select unhex('41')")
-    testFoldConst("select unhex('4142')")
-    testFoldConst("select unhex('')")
-    testFoldConst("select unhex(NULL)")
-
-    testFoldConst("select instr(\"abc\", \"b\")")
-    testFoldConst("select instr(\"abc\", \"d\")")
-    testFoldConst("select instr(\"abc\", null)")
-    testFoldConst("select instr(null, \"a\")")
-    testFoldConst("SELECT instr('foobar', '')")
-    testFoldConst("SELECT instr('上海天津北京杭州', '北京')")
-
-    testFoldConst("SELECT lcase(\"AbC123\")")
-    testFoldConst("SELECT lower(\"AbC123\")")
-
-    testFoldConst("SELECT initcap(\"AbC123abc abc.abc,?|abc\")")
-
-    testFoldConst("select left(\"Hello doris\",5)")
-    testFoldConst("select right(\"Hello doris\",5)")
-
-    testFoldConst("select length(\"abc\")")
-
-    testFoldConst("SELECT LOCATE('bar', 'foobarbar')")
-    testFoldConst("SELECT LOCATE('xbar', 'foobar')")
-    testFoldConst("SELECT LOCATE('', 'foobar')")
-    testFoldConst("SELECT LOCATE('北京', '上海天津北京杭州')")
-
-    testFoldConst("SELECT lpad(\"hi\", 5, \"xy\")")
-    testFoldConst("SELECT lpad(\"hi\", 1, \"xy\")")
-    testFoldConst("SELECT rpad(\"hi\", 5, \"xy\")")
-    testFoldConst("SELECT rpad(\"hi\", 1, \"xy\")")
-
-    testFoldConst("SELECT ltrim('   ab d')")
-
-    testFoldConst("select money_format(17014116)")
-    testFoldConst("select money_format(1123.456)")
+    
+    // ifnull
+    testFoldConst("select ifnull(null,3)")
+    testFoldConst("select ifnull(3,null)")
+    testFoldConst("select ifnull(null,null)")
+    
+    // initcap
+    testFoldConst("select initcap('AbC123abc abc.abc,?|abc')")
+    testFoldConst("select initcap(cast('AbC123abc abc.abc,?|abc' as string))")
+    testFoldConst("select initcap(cast('hello world' as string))")
+    testFoldConst("select initcap('hello world')")
+    testFoldConst("select initcap(' hello world')")
+    testFoldConst("select initcap('こんにちは')")
+    testFoldConst("select initcap('上海天津北京杭州')")
+    
+    // instr
+    testFoldConst("select instr('上海天津北京杭州', '北京')")
+    testFoldConst("select instr('abc', 'b')")
+    testFoldConst("select instr('abc', 'd')")
+    testFoldConst("select instr('abc', 'abcd')")
+    testFoldConst("select instr('abc', null)")
+    testFoldConst("select instr(cast('Hello' as string), cast('World' as string))")
+    testFoldConst("select instr(cast('Hello World' as string), cast('World' as string))")
+    testFoldConst("select instr('foobar', '')")
+    testFoldConst("select instr('Hello', 'World')")
+    testFoldConst("select instr('Hello World', 'World')")
+    testFoldConst("select instr(null, 'a')")
+    testFoldConst("select instr(NULL, cast('a' as string))")
+    testFoldConst("select instr('', 'World')")
+    
+    // lcase
+    testFoldConst("select lcase('AbC123')")
+    testFoldConst("select lcase(cast('AbC123' as string))")
+    testFoldConst("select lcase('上海天津北京杭州')")
+    testFoldConst("select lcase('こんにちは')")
+    
+    // left
+    testFoldConst("select left(CAST('good morning' AS STRING), 120)")
+    testFoldConst("select left(CAST('good morning' AS STRING), -5)")
+    testFoldConst("select left(CAST('good morning' AS STRING), NULL)")
+    testFoldConst("select left(cast('Hello' as string), 10)")
+    testFoldConst("select left(cast('Hello doris' as string), 5)")
+    testFoldConst("select left(CAST('Hello doris' AS STRING), 5)")
+    testFoldConst("select left(cast('Hello World' as string), 5)")
+    testFoldConst("select left(CAST(NULL AS STRING), 1)")
+    testFoldConst("select left('good morning', 120)")
+    testFoldConst("select left('good morning', -5)")
+    testFoldConst("select left('good morning', NULL)")
+    testFoldConst("select left('Hello', 10)")
+    testFoldConst("select left('', 10)")
+    testFoldConst("select left(' Hello', 10)")
+    testFoldConst("select left('Hello doris', 5)")
+    testFoldConst("select left('Hello doris',5)")
+    testFoldConst("select left('Hello World', 5)")
+    testFoldConst("select left(NULL, 1)")
+    testFoldConst("select left('上海天津北京杭州', 5)")
+    testFoldConst("select left('上海天津北京杭州', -5)")
+    testFoldConst("select left('上海天津北京杭州', 0)")
+    
+    // length
+    testFoldConst("select length('你')")
+    testFoldConst("select length('abc')")
+    testFoldConst("select length(cast('abc' as string))")
+    testFoldConst("select length(cast('Hello World' as string))")
+    testFoldConst("select length('Hello World')")
+    testFoldConst("select length('')")
+    testFoldConst("select length(' Hello World')")
+    testFoldConst("select length(space(10))")
+
+    // locate
+    testFoldConst("select locate('北京', '上海天津北京杭州')")
+    testFoldConst("select locate('上海天津北京杭州', '北京')")
+    testFoldConst("select locate('bar', 'foobarbar')")
+    testFoldConst("select locate(cast('北京' as string), cast('上海天津北京杭州' as string))")
+    testFoldConst("select locate(cast('' as string), cast('foobar' as string))")
+    testFoldConst("select locate(cast('bar' as string), cast('foobarbar' as string))")
+    testFoldConst("select locate(cast('World' as string), cast('Hello' as string))")
+    testFoldConst("select locate(cast('World' as string), cast('Hello World' as string))")
+    testFoldConst("select locate(cast('xbar' as string), cast('foobar' as string))")
+    testFoldConst("select locate('', 'foobar')")
+    testFoldConst("select locate('World', 'Hello')")
+    testFoldConst("select locate('World', 'Hello World')")
+    testFoldConst("select locate('xbar', 'foobar')")
+    testFoldConst("select locate('北京', '上海天津北京杭州', 4)")
+    testFoldConst("select locate('北京', '上海天津北京杭州', 5)")
+    testFoldConst("select locate('北京', '上海天津北京杭州', -4)")
+    testFoldConst("select locate('北京', '上海天津北京杭州', -5)")
+    testFoldConst("select locate('2', '   123  ', 1)")
+    
+    // lower
+    testFoldConst("select lower('AbC123')")
+    testFoldConst("select lower(cast('AbC123' as string))")
+    testFoldConst("select lower(cast('Hello World' as string))")
+    testFoldConst("select lower('Hello World')")
+    
+    // lpad
+    testFoldConst("select lpad(cast('hi' as string), 1, cast('xy' as string))")
+    testFoldConst("select lpad(cast('hi' as string), 5, cast('xy' as string))")
+    testFoldConst("select lpad('hi', 1, 'xy')")
+    testFoldConst("select lpad('hi', 5, 'xy')")
+    testFoldConst("select lpad('hi', 1, '')")
+    testFoldConst("select lpad('', 1, 'xy')")
+    testFoldConst("select lpad('hi', 1, ' ')")
+    testFoldConst("select lpad(' ', 1, 'xy')")
+    testFoldConst("select lpad(cast('北京' as string), 1, cast('杭州' as string))")
+    testFoldConst("select lpad(cast('北京' as string), 5, cast('杭州' as string))")
+    
+    // ltrim
+    testFoldConst("select ltrim(' 11111', 11)")
+    testFoldConst("select ltrim('11111 ', 11)")
+    testFoldConst("select ltrim('   ab d')")
+    testFoldConst("select ltrim(cast(' 11111' as string), cast(11 as string))")
+    testFoldConst("select ltrim(cast('11111 ' as string), cast(11 as string))")
+    testFoldConst("select ltrim(cast('   ab d' as string))")
+    testFoldConst("select ltrim(cast('Hello' as string))")
+    testFoldConst("select ltrim(cast('  Hello World  ' as string))")
+    testFoldConst("select ltrim('Hello')")
+    testFoldConst("select ltrim('  Hello World  ')")
+    testFoldConst("select ltrim('  上海天津北京杭州  ')")
+    
+    // md5
+    testFoldConst("select md5(cast('Hello World' as string))")
+    testFoldConst("select md5('Hello World')")
+    testFoldConst("select md5(' Hello World')")
+    testFoldConst("select md5('Hello World ')")
+    testFoldConst("select md5('')")
+    testFoldConst("select md5('こんにちは')")
+    testFoldConst("select md5sum('Hello World')")
+    testFoldConst("select md5sum('こんにちは')")
+    testFoldConst("select md5sum('===*+-')")
+
+    // money_format
     testFoldConst("select money_format(1123.4)")
+    testFoldConst("select money_format(1123.456)")
+    testFoldConst("select money_format(17014116)")
     testFoldConst("select money_format(truncate(1000,10))")
-
-    testFoldConst("select null_or_empty(null)")
-    testFoldConst("select null_or_empty(\"\")")
-    testFoldConst("select null_or_empty(\"a\")")
-
+    testFoldConst("select money_format(-1123.4)")
+    testFoldConst("select money_format(-1123.456)")
+    testFoldConst("select money_format(-17014116)")
+    testFoldConst("select money_format(-truncate(1000,10))")
+    
+    // not_null_or_empty
+    testFoldConst("select not_null_or_empty('')")
+    testFoldConst("select not_null_or_empty('a')")
+    testFoldConst("select not_null_or_empty(cast('a' as string))")
+    testFoldConst("select not_null_or_empty(cast('' as string))")
+    testFoldConst("select not_null_or_empty(cast('       ' as string))")
     testFoldConst("select not_null_or_empty(null)")
-    testFoldConst("select not_null_or_empty(\"\")")
-    testFoldConst("select not_null_or_empty(\"a\")")
-
-    testFoldConst("SELECT repeat(\"a\", 3)")
-    testFoldConst("SELECT repeat(\"a\", -1)")
-    testFoldConst("SELECT repeat(\"a\", 0)")
-    testFoldConst("SELECT repeat(\"a\",null)")
-    testFoldConst("SELECT repeat(null,1)")
-
-    testFoldConst("select replace(\"https://doris.apache.org:9090\", \":9090\", \"\")")
-    testFoldConst("select replace(\"https://doris.apache.org:9090\", \"\", \"new_str\")")
-
-    testFoldConst("SELECT REVERSE('hello')")
-
-    testFoldConst("select split_part('hello world', ' ', 1)")
-    testFoldConst("select split_part('hello world', ' ', 2)")
-    testFoldConst("select split_part('hello world', ' ', 0)")
-    testFoldConst("select split_part('hello world', ' ', -1)")
-    testFoldConst("select split_part('hello world', ' ', -2)")
-    testFoldConst("select split_part('hello world', ' ', -3)")
+    testFoldConst("select not_null_or_empty(NULL)")
+    testFoldConst("select not_null_or_empty('\b')")
+    testFoldConst("select not_null_or_empty(' \b')")
+    
+    // null_or_empty
+    testFoldConst("select null_or_empty('')")
+    testFoldConst("select null_or_empty('a')")
+    testFoldConst("select null_or_empty(cast('a' as string))")
+    testFoldConst("select null_or_empty(cast('' as string))")
+    testFoldConst("select null_or_empty(null)")
+    testFoldConst("select null_or_empty(NULL)")
+    testFoldConst("select null_or_empty('\b')")
+    testFoldConst("select null_or_empty(' \b')")
+    
+    // overlay
+    testFoldConst("select overlay('abcdef', 3, 2, '123')")
+    testFoldConst("select overlay('abcdef', 10, 20, '123')")
+    testFoldConst("select overlay(null, 3, 2, '123')")
+    testFoldConst("select overlay('abcdef', 3, 2, null)")
+    testFoldConst("select overlay(cast('abcdef' as string), 3, 2, cast('123' as string))")
+    testFoldConst("select overlay('PRD-1234-5678', 5, 4, '9876')")
+    // be has bug
+//    testFoldConst("select overlay('こんにちは', 1, 2, 'にちは')")
+    
+    // parse_url
+    testFoldConst("select parse_url(cast('http://www.example.com/path?query=abc' as string), cast('HOST' as string))")
+    testFoldConst("select parse_url('http://www.example.com/path?query=abc', 'HOST')")
+    testFoldConst("select parse_url('http://www.example.com/path?query=abc', 'QUERY')")
+    testFoldConst("select parse_url('http://www.example.com/path?query=こんにちは', 'QUERY')")
+    testFoldConst("select parse_url(\"http://www.example.com/path?query=a\b\'\", 'QUERY')")
+    testFoldConst("select parse_url(\"http://www.example.com/path.query=a\b\'\", 'QUERY')")
+
+    // repeat
+    testFoldConst("select repeat('a', 0)")
+    testFoldConst("select repeat('a', -1)")
+    testFoldConst("select repeat('a', 3)")
+    testFoldConst("select repeat('a',null)")
+    testFoldConst("select repeat(cast('a' as string), 0)")
+    testFoldConst("select repeat(cast('a' as string), -1)")
+    testFoldConst("select repeat(cast('a' as string), 3)")
+    testFoldConst("select repeat(cast('Hello' as string), -3)")
+    testFoldConst("select repeat(cast('Hello' as string), 3)")
+    testFoldConst("select repeat('Hello', -3)")
+    testFoldConst("select repeat('Hello', 3)")
+    testFoldConst("select repeat(NULL, 1)")
+    testFoldConst("select repeat('', 3)")
+    testFoldConst("select repeat(' ', 3)")
+    testFoldConst("select repeat('前进',4)")
+    
+    // replace
+    testFoldConst("select replace(cast('Hello World' as string), '', cast('Everyone' as string))")
+    testFoldConst("select replace(cast('Hello World' as string), cast('World' as string), '')")
+    testFoldConst("select replace(cast('Hello World' as string), cast('World' as string), cast('Everyone' as string))")
+    testFoldConst("select replace(cast('https://doris.apache.org:9090' as string), cast(':9090' as string), cast('' as string))")
+    testFoldConst("select replace(cast('https://doris.apache.org:9090' as string), cast('' as string), cast('new_str' as string))")
+    testFoldConst("select replace('Hello World', '', 'Everyone')")
+    testFoldConst("select replace('Hello World', 'World', '')")
+    testFoldConst("select replace('Hello World', 'World', 'Everyone')")
+    testFoldConst("select replace('https://doris.apache.org:9090', ':9090', '')")
+    testFoldConst("select replace('https://doris.apache.org:9090', '', 'new_str')")
+    testFoldConst("select replace('https://doris.apache.org:9090', './*', 'new_str')")
+    
+    // reverse
+    testFoldConst("select reverse('Hello')")
+    testFoldConst("select reverse('')")
+    testFoldConst("select reverse('こんにちは')")
+    
+    // right
+    testFoldConst("select right(CAST('good morning' AS STRING), NULL)")
+    testFoldConst("select right(cast('Hello' as string), 10)")
+    testFoldConst("select right(CAST('Hello doris' AS STRING), 120)")
+    testFoldConst("select right(cast('Hello doris' as string), 5)")
+    testFoldConst("select right(CAST('Hello doris' AS STRING), 5)")
+    testFoldConst("select right(CAST('Hello doris' AS STRING), -6)")
+    testFoldConst("select right(cast('Hello World' as string), 5)")
+    testFoldConst("select right(CAST(NULL AS STRING), 1)")
+    testFoldConst("select right('good morning', NULL)")
+    testFoldConst("select right('Hello', 10)")
+    testFoldConst("select right('Hello doris', 120)")
+    testFoldConst("select right('Hello doris', 5)")
+    testFoldConst("select right('Hello doris',5)")
+    testFoldConst("select right('Hello doris', -6)")
+    testFoldConst("select right('Hello World', 5)")
+    testFoldConst("select right('Hello World', 0)")
+    testFoldConst("select right(NULL, 1)")
+    
+    // rpad
+    testFoldConst("select rpad(cast('hi' as string), 1, cast('xy' as string))")
+    testFoldConst("select rpad(cast('hi' as string), 5, cast('xy' as string))")
+    testFoldConst("select rpad('hi', 1, 'xy')")
+    testFoldConst("select rpad('hi', 5, 'xy')")
+    
+    // rtrim
+    testFoldConst("select rtrim(' 11111', 11)")
+    testFoldConst("select rtrim('11111 ', 11)")
+    testFoldConst("select rtrim(cast(' 11111' as string), cast(11 as string))")
+    testFoldConst("select rtrim(cast('11111 ' as string), cast(11 as string))")
+    testFoldConst("select rtrim(cast('Hello' as string))")
+    testFoldConst("select rtrim(cast('  Hello World  ' as string))")
+    testFoldConst("select rtrim('Hello')")
+    testFoldConst("select rtrim('  Hello World  ')")
+    
+    // space
+    testFoldConst("select space(-5)")
+    testFoldConst("select space(5)")
+    testFoldConst("select space(0)")
+
+    // split_by_string
+    testFoldConst("select split_by_string('::', 'abc')")
+    testFoldConst("select split_by_string('a::b::c', '::')")
+    testFoldConst("select split_by_string(cast('a::b::c' as string), cast('::' as string))")
+    testFoldConst("select split_by_string(cast('abc' as string), cast('::' as string))")
+    testFoldConst("select split_by_string('上海天津北京杭州', '北')")
+    testFoldConst("select split_by_string('abccccc', 'c')")
+    
+    // split_part
+    testFoldConst("select split_part('a,b,c', ',', -1)")
     testFoldConst("select split_part('abc##123###xyz', '##', 0)")
-    testFoldConst("select split_part('abc##123###xyz', '##', 1)")
-    testFoldConst("select split_part('abc##123###xyz', '##', 3)")
-    testFoldConst("select split_part('abc##123###xyz', '##', 5)")
     testFoldConst("select split_part('abc##123###xyz', '##', -1)")
+    testFoldConst("select split_part('abc##123###xyz', '##', 1)")
     testFoldConst("select split_part('abc##123###xyz', '##', -2)")
+    testFoldConst("select split_part('abc##123###xyz', '##', 3)")
     testFoldConst("select split_part('abc##123###xyz', '##', -4)")
-
-    testFoldConst("select starts_with(\"hello world\",\"hello\")")
-    testFoldConst("select starts_with(\"hello world\",\"world\")")
-    testFoldConst("select starts_with(\"hello world\",null)")
-
+    testFoldConst("select split_part('abc##123###xyz', '##', 5)")
+    testFoldConst("select split_part('a,b,c', ',', 2)")
+    testFoldConst("select split_part('a,b,c', ',', 5)")
+    testFoldConst("select split_part(cast('a,b,c' as string), cast(',' as string), -1)")
+    testFoldConst("select split_part(cast('a,b,c' as string), cast(',' as string), 2)")
+    testFoldConst("select split_part(cast('a,b,c' as string), cast(',' as string), 5)")
+    testFoldConst("select split_part(cast('hello world' as string), cast(' ' as string), 1)")
+    testFoldConst("select split_part(cast('hello world' as string), cast(' ' as string), 2)")
+    testFoldConst("select split_part(cast('hello world' as string), cast(' ' as string), 3)")
+    testFoldConst("select split_part('hello world', ' ', 0)")
+    testFoldConst("select split_part('hello world', ' ', -1)")
+    testFoldConst("select split_part('hello world', ' ', 1)")
+    testFoldConst("select split_part('hello world', ' ', -2)")
+    testFoldConst("select split_part('hello world', ' ', 2)")
+    testFoldConst("select split_part('hello world', ' ', -3)")
+    testFoldConst("SELECT split_part('哈哈哈AAA','A', -5)")
+    testFoldConst("SELECT split_part('哈哈哈AAA','A', -4)")
+    testFoldConst("SELECT split_part('哈哈哈AAA','A', -3)")
+    testFoldConst("SELECT split_part('哈哈哈AAA','A', -2)")
+    testFoldConst("SELECT split_part('哈哈哈AAA','A', -1)")
+    testFoldConst("SELECT split_part('哈哈哈AAA','A', 0)")
+    testFoldConst("SELECT split_part('哈哈哈AAA','A', 1)")
+    testFoldConst("SELECT split_part('哈哈哈AAA','A', 2)")
+    testFoldConst("SELECT split_part('哈哈哈AAA','A', 3)")
+    testFoldConst("SELECT split_part('哈哈哈AAA','A', 4)")
+    testFoldConst("SELECT split_part('哈哈哈AAA','A', 5)")
+    testFoldConst("SELECT split_part('哈哈哈AA+','A', -4)")
+    testFoldConst("SELECT split_part('哈哈哈AA+','A', -3)")
+    testFoldConst("SELECT split_part('哈哈哈AA+','A', -2)")
+    testFoldConst("SELECT split_part('哈哈哈AA+','A', -1)")
+    testFoldConst("SELECT split_part('哈哈哈AA+','A', 0)")
+    testFoldConst("SELECT split_part('哈哈哈AA+','A', 1)")
+    testFoldConst("SELECT split_part('哈哈哈AA+','A', 2)")
+    testFoldConst("SELECT split_part('哈哈哈AA+','A', 3)")
+    testFoldConst("SELECT split_part('哈哈哈AA+','A', 4)")
+    
+    // starts_with
+    testFoldConst("select starts_with('hello world','hello')")
+    testFoldConst("select starts_with('hello world',null)")
+    testFoldConst("select starts_with('hello world','world')")
+    testFoldConst("select starts_with(' hello world','world')")
+    testFoldConst("select starts_with('上海天津北京杭州','上海')")
+    testFoldConst("select starts_with('上海天津北京杭州','北京')")
+    
+    // strcmp
+    testFoldConst("select strcmp('a', 'abc')")
+    testFoldConst("select strcmp('abc', 'abc')")
+    testFoldConst("select strcmp('abc', 'abc')")
+    testFoldConst("select strcmp('abc', 'abd')")
+    testFoldConst("select strcmp('abcd', 'abc')")
+    testFoldConst("select strcmp('abc', NULL)")
+    testFoldConst("select strcmp(CAST('a' AS STRING), CAST('abc' AS STRING))")
+    testFoldConst("select strcmp(cast('abc' as string), cast('abc' as string))")
+    testFoldConst("select strcmp(CAST('abc' AS STRING), CAST('abc' AS STRING))")
+    testFoldConst("select strcmp(cast('abc' as string), cast('abd' as string))")
+    testFoldConst("select strcmp(cast('abc' as string), NULL)")
+    testFoldConst("select strcmp(CAST('abcd' AS STRING), CAST('abc' AS STRING))")
+    
+    // strleft
+    testFoldConst("select strleft('good morning', 120)")
+    testFoldConst("select strleft('good morning', -5)")
+    testFoldConst("select strleft('good morning', NULL)")
+    testFoldConst("select strleft('Hello doris', 5)")
+    testFoldConst("select strleft('Hello World', 5)")
+    testFoldConst("select strleft(' Hello World', 5)")
+    testFoldConst("select strleft('Hello World ', 50)")
     testFoldConst("select strleft(NULL, 1)")
-    testFoldConst("select strleft(\"good morning\", NULL)")
-    testFoldConst("select left(NULL, 1)")
-    testFoldConst("select left(\"good morning\", NULL)")
-    testFoldConst("select strleft(\"Hello doris\", 5)")
-    testFoldConst("select left(\"Hello doris\", 5)")
+    
+    // strright
+    testFoldConst("select strright('good morning', NULL)")
+    testFoldConst("select strright('Hello doris', 120)")
+    testFoldConst("select strright('Hello doris', -5)")
+    testFoldConst("select strright('Hello doris', 5)")
+    testFoldConst("select strright('Hello World', 5)")
+    testFoldConst("select strright(' Hello World', 5)")
+    testFoldConst("select strright('Hello World  ', 5)")
     testFoldConst("select strright(NULL, 1)")
-    testFoldConst("select strright(\"good morning\", NULL)")
-    testFoldConst("select right(NULL, 1)")
-    testFoldConst("select right(\"good morning\", NULL)")
-    testFoldConst("select strright(\"Hello doris\", 5)")
-    testFoldConst("select right(\"Hello doris\", 5)")
-    testFoldConst("select strleft(\"good morning\", 120)")
-    testFoldConst("select strleft(\"good morning\", -5)")
-    testFoldConst("select strright(\"Hello doris\", 120)")
-    testFoldConst("select strright(\"Hello doris\", -5)")
-    testFoldConst("select left(\"good morning\", 120)")
-    testFoldConst("select left(\"good morning\", -5)")
-    testFoldConst("select right(\"Hello doris\", 120)")
-    testFoldConst("select right(\"Hello doris\", -6)")
-
-    testFoldConst("select substring('abc1', 2)")
-    testFoldConst("select substring('abc1', -2)")
-    testFoldConst("select substring('abc1', 5)")
-    testFoldConst("select substring('abc1def', 2, 2)")
-    testFoldConst("select substring('abcdef',3,-1)")
-    testFoldConst("select substring('abcdef',-3,-1)")
-    testFoldConst("select substring('abcdef',10,1)")
-
-    testFoldConst("select substr('a',3,1)")
-    testFoldConst("select substr('a',2,1)")
-    testFoldConst("select substr('a',1,1)")
+    
+    // sub_replace
+    testFoldConst("select sub_replace(CAST('doris' AS STRING), CAST('***' AS STRING), 1, 2)")
+    testFoldConst("select sub_replace(CAST('doris' AS STRING), CAST('***' AS STRING), 1, 2)")
+    testFoldConst("select sub_replace(CAST('this is origin str' AS STRING), CAST('NEW-STR' AS STRING), 1)")
+    testFoldConst("select sub_replace(CAST('this is origin str' AS STRING), CAST('NEW-STR' AS STRING), 1)")
+    testFoldConst("select sub_replace('doris','***',1,2)")
+    testFoldConst("select sub_replace('doris','***',1,2)")
+    testFoldConst("select sub_replace('this is origin str','NEW-STR',1)")
+    testFoldConst("select sub_replace('this is origin str','NEW-STR',1)")
+    testFoldConst("select sub_replace(CAST('doris' AS STRING), CAST('***' AS STRING), -1, 2)")
+    testFoldConst("select sub_replace('上海天津北京杭州', '天津', 3, 4)")
+    testFoldConst("select sub_replace('上海天津北京杭州', '天津', 30, 4)")
+    
+    // substr
     testFoldConst("select substr('a',0,1)")
     testFoldConst("select substr('a',-1,1)")
+    testFoldConst("select substr('a',1,1)")
     testFoldConst("select substr('a',-2,1)")
+    testFoldConst("select substr('a',2,1)")
     testFoldConst("select substr('a',-3,1)")
-    testFoldConst("select substr('abcdef',3,-1)")
+    testFoldConst("select substr('a',3,1)")
     testFoldConst("select substr('abcdef',-3,-1)")
+    testFoldConst("select substr('abcdef',3,-1)")
+    testFoldConst("select substr('',3,-1)")
+    testFoldConst("select substr('abcdef',3,10)")
 
-    testFoldConst("select sub_replace(\"this is origin str\",\"NEW-STR\",1)")
-    testFoldConst("select sub_replace(\"doris\",\"***\",1,2)")
-
-    testFoldConst("select substring_index(\"hello world\", \" \", 1)")
-    testFoldConst("select substring_index(\"hello world\", \" \", 2)")
-    testFoldConst("select substring_index(\"hello world\", \" \", 3)")
-    testFoldConst("select substring_index(\"hello world\", \" \", -1)")
-    testFoldConst("select substring_index(\"hello world\", \" \", -2)")
-    testFoldConst("select substring_index(\"hello world\", \" \", -3)")
-    testFoldConst("select substring_index(\"prefix__string2\", \"__\", 2)")
-    testFoldConst("select substring_index(\"prefix__string2\", \"_\", 2)")
-    testFoldConst("select substring_index(\"prefix_string2\", \"__\", 1)")
-    testFoldConst("select substring_index(null, \"__\", 1)")
-    testFoldConst("select substring_index(\"prefix_string\", null, 1)")
-    testFoldConst("select substring_index(\"prefix_string\", \"_\", null)")
-    testFoldConst("select substring_index(\"prefix_string\", \"__\", -1)")
-
-    testFoldConst("select elt(0, \"hello\", \"doris\")")
-    testFoldConst("select elt(1, \"hello\", \"doris\")")
-    testFoldConst("select elt(2, \"hello\", \"doris\")")
-    testFoldConst("select elt(3, \"hello\", \"doris\")")
-
-    testFoldConst("select sub_replace(\"this is origin str\",\"NEW-STR\",1)")
-    testFoldConst("select sub_replace(\"doris\",\"***\",1,2)")
-
-    testFoldConst("select strcmp('a', 'abc')")
-    testFoldConst("select strcmp('abc', 'abc')")
-    testFoldConst("select strcmp('abcd', 'abc')")
-
-    testFoldConst("SELECT Concat(cast('Hello' as string), cast(' ' as string), cast('World' as string))")
-    testFoldConst("SELECT Substring(cast('Hello World' as string), 1, 5)")
-    testFoldConst("SELECT Substring(cast('1' as string), 1, 1)")
-    testFoldConst("SELECT 100, cast('abc' as string), Substring(cast('abc' as string), 1, 2), Substring(Substring(cast('abcdefg' as string), 4, 3), 1, 2), null")
-    testFoldConst("SELECT Length(cast('Hello World' as string))")
-    testFoldConst("SELECT Lower(cast('Hello World' as string))")
-    testFoldConst("SELECT Upper(cast('Hello World' as string))")
-    testFoldConst("SELECT Trim(cast('  Hello World  ' as string))")
-    testFoldConst("SELECT Trim(cast('11111' as string), cast(11 as string))")
-    testFoldConst("SELECT Ltrim(cast('  Hello World  ' as string))")
-    testFoldConst("SELECT LTrim(cast(' 11111' as string), cast(11 as string))")
-    testFoldConst("SELECT LTrim(cast('11111 ' as string), cast(11 as string))")
-    testFoldConst("SELECT Rtrim(cast('  Hello World  ' as string))")
-    testFoldConst("SELECT RTrim(cast('11111 ' as string), cast(11 as string))")
-    testFoldConst("SELECT RTrim(cast(' 11111' as string), cast(11 as string))")
-    testFoldConst("SELECT Replace(cast('Hello World' as string), cast('World' as string), cast('Everyone' as string))")
-    testFoldConst("SELECT Left(cast('Hello World' as string), 5)")
-    testFoldConst("SELECT Right(cast('Hello World' as string), 5)")
-    testFoldConst("SELECT Locate(cast('World' as string), cast('Hello World' as string))")
-    testFoldConst("SELECT Instr(cast('Hello World' as string), cast('World' as string))")
-    testFoldConst("SELECT Ascii(cast('A' as string))")
-    testFoldConst("SELECT Bin(5)")
-    testFoldConst("SELECT Hex(255)")
-    testFoldConst("SELECT Unhex(cast('FF' as string))")
-//    testFoldConst("SELECT Concat_Ws(cast('-' as string), cast('2024' as string), cast('09' as string), cast('02' as string))")
-    testFoldConst("SELECT Char(65)")
-    testFoldConst("SELECT Character_Length(cast('Hello World' as string))")
-    testFoldConst("SELECT Initcap(cast('hello world' as string))")
-    testFoldConst("SELECT Md5(cast('Hello World' as string))")
-//    testFoldConst("SELECT Md5Sum(cast('Hello World' as string))")
-// testFoldConst("SELECT JsonExtract(cast('{\"key\": \"value\"}' as string), cast('$.key' as string))")
-// testFoldConst("SELECT JsonbExtractString(cast('{\"key\": \"value\"}' as string), cast('$.key' as string))")
-// testFoldConst("SELECT JsonContains(cast('{\"key\": \"value\"}' as string), cast('\"key\"' as string))")
-// testFoldConst("SELECT JsonLength(cast('{\"key1\": \"value1\", \"key2\": \"value2\"}' as string))")
-// testFoldConst("SELECT JsonObject(cast('key' as string), cast('value' as string))")
-// testFoldConst("SELECT JsonArray(cast('value1' as string), cast('value2' as string))")
-// testFoldConst("SELECT JsonKeys(cast('{\"key1\": \"value1\", \"key2\": \"value2\"}' as string))")
-// testFoldConst("SELECT JsonInsert(cast('{\"key1\": \"value1\"}' as string), cast('$.key2' as string), cast('value2' as string))")
-// testFoldConst("SELECT JsonReplace(cast('{\"key1\": \"value1\"}' as string), cast('$.key1' as string), cast('new_value' as string))")
-// testFoldConst("SELECT JsonSet(cast('{\"key1\": \"value1\"}' as string), cast('$.key2' as string), cast('value2' as string))")
-// testFoldConst("SELECT Json_Quote(cast('Hello World' as string))")
-// testFoldConst("SELECT Json_UnQuote(cast('\"Hello World\"' as string))")
-//    testFoldConst("SELECT Field(cast('b' as string), cast('a' as string), cast('b' as string), cast('c' as string))")
-    testFoldConst("SELECT Find_In_Set(cast('b' as string), cast('a,b,c' as string))")
-    testFoldConst("SELECT Repeat(cast('Hello' as string), 3)")
-    testFoldConst("SELECT Reverse(cast('Hello' as string))")
-    testFoldConst("SELECT length(Space(10))")
-// testFoldConst("SELECT Split_By_Char(cast('a,b,c' as string), cast(',' as string))")  has bug in be execution
-    testFoldConst("SELECT Split_By_String(cast('a::b::c' as string), cast('::' as string))")
-    testFoldConst("SELECT Split_Part(cast('a,b,c' as string), cast(',' as string), 2)")
-    testFoldConst("SELECT Substring_Index(cast('a,b,c' as string), cast(',' as string), 2)")
-    testFoldConst("SELECT Strcmp(cast('abc' as string), cast('abd' as string))")
-    testFoldConst("SELECT StrLeft(cast('Hello World' as string), 5)")
-    testFoldConst("SELECT StrRight(cast('Hello World' as string), 5)")
-    testFoldConst("SELECT Overlay(cast('abcdef' as string), cast('123' as string), 3, 2)")
-    testFoldConst("SELECT Parse_Url(cast('http://www.example.com/path?query=abc' as string), cast('HOST' as string))")
-    testFoldConst("SELECT Url_Decode(cast('+Hello+World+' as string))")
-    testFoldConst("SELECT Url_Encode(cast(' Hello World ' as string))")
-
-// Substring with negative start index
-// Expected behavior: Depending on the SQL engine, might return an empty string or error.
-    testFoldConst("SELECT Substring(cast('Hello World' as string), -1, 5)")
-
-// Substring with length exceeding the string length
-// Expected behavior: Return 'Hello' as the length exceeds the string length.
-    testFoldConst("SELECT Substring(cast('Hello' as string), 1, 10)")
-
-// Left with length greater than string length
-// Expected behavior: Return 'Hello'.
-    testFoldConst("SELECT Left(cast('Hello' as string), 10)")
-
-// Right with length greater than string length
-// Expected behavior: Return 'Hello'.
-    testFoldConst("SELECT Right(cast('Hello' as string), 10)")
-
-// SplitPart with part number greater than the number of parts
-// Expected behavior: Return an empty string or error.
-    testFoldConst("SELECT Split_Part(cast('a,b,c' as string), cast(',' as string), 5)")
-
-// SplitPart with negative part number
-// Expected behavior: Return an empty string or error.
-    testFoldConst("SELECT Split_Part(cast('a,b,c' as string), cast(',' as string), -1)")
-
-// Locate with the substring not present
-// Expected behavior: Return 0 as 'World' is not found.
-    testFoldConst("SELECT Locate(cast('World' as string), cast('Hello' as string))")
-
-// Instr with the substring not present
-// Expected behavior: Return 0 as 'World' is not found.
-    testFoldConst("SELECT Instr(cast('Hello' as string), cast('World' as string))")
-
-// Replace with an empty search string
-// Expected behavior: Some SQL engines may treat this as a no-op, others might throw an error.
-    testFoldConst("SELECT Replace(cast('Hello World' as string), '', cast('Everyone' as string))")
-
-// Replace with an empty replacement string
-// Expected behavior: Return 'Hello '.
-    testFoldConst("SELECT Replace(cast('Hello World' as string), cast('World' as string), '')")
-
-// Concat with NULL values
-// Expected behavior: Depending on the SQL engine, may return 'HelloWorld' or NULL.
-    testFoldConst("SELECT Concat(cast('Hello' as string), NULL, cast('World' as string))")
-
-// Ltrim with a string that has no leading spaces
-// Expected behavior: Return 'Hello'.
-    testFoldConst("SELECT Ltrim(cast('Hello' as string))")
-
-// Rtrim with a string that has no trailing spaces
-// Expected behavior: Return 'Hello'.
-    testFoldConst("SELECT Rtrim(cast('Hello' as string))")
-
-// Testing JSON Length function with a non-JSON string
-    testFoldConst("SELECT Json_Length(cast('Hello World' as string))")
-
-// Field with a value not present in the list
-//    testFoldConst("SELECT Field(cast('d' as string), cast('a' as string), cast('b' as string), cast('c' as string))")
-
-// FindInSet with a value not present in the set
-    testFoldConst("SELECT Find_In_Set(cast('d' as string), cast('a,b,c' as string))")
-
-// Repeat with a negative repeat count
-    testFoldConst("SELECT Repeat(cast('Hello' as string), -3)")
-
-// Space with a negative number of spaces
-    testFoldConst("SELECT Space(-5)")
-
-// SplitByChar with a delimiter not present in the string
-//    testFoldConst("SELECT Split_By_Char(cast('abc' as string), cast(',' as string))")
-
-// SplitByString with a delimiter not present in the string
-    testFoldConst("SELECT Split_By_String(cast('abc' as string), cast('::' as string))")
-
-// Strcmp with two identical strings
-    testFoldConst("SELECT Strcmp(cast('abc' as string), cast('abc' as string))")
-
-// Strcmp with a null string
-    testFoldConst("SELECT Strcmp(cast('abc' as string), NULL)")
-
-// Hex with a negative number
-    testFoldConst("SELECT Hex(-255)")
-
-// Unhex with an invalid hexadecimal string
-    testFoldConst("SELECT Unhex(cast('GHIJ' as string))")
-
-// UrlDecode with an invalid percent-encoded string
-    // testFoldConst("SELECT Url_Decode(cast('%ZZHello%20World' as string))")
-
-// Additional function tests
-    testFoldConst("SELECT Elt(0, cast('hello' as string), cast('doris' as string))")
-    testFoldConst("SELECT Elt(1, cast('hello' as string), cast('doris' as string))")
-    testFoldConst("SELECT Elt(2, cast('hello' as string), cast('doris' as string))")
-    testFoldConst("SELECT Elt(3, cast('hello' as string), cast('doris' as string))")
-    testFoldConst("SELECT Append_Trailing_Char_If_Absent(cast('a' as string), cast('c' as string))")
-    testFoldConst("SELECT Append_Trailing_Char_If_Absent(cast('ac' as string), cast('c' as string))")
-    testFoldConst("SELECT Ascii(cast('1' as string))")
-    testFoldConst("SELECT Ascii(cast('a' as string))")
-    testFoldConst("SELECT Ascii(cast('A' as string))")
-    testFoldConst("SELECT Ascii(cast('!' as string))")
-    testFoldConst("SELECT Bit_Length(cast('abc' as string))")
-    testFoldConst("SELECT Char_Length(cast('abc' as string))")
-    testFoldConst("SELECT Concat(cast('a' as string), cast('b' as string))")
-    testFoldConst("SELECT Concat(cast('a' as string), cast('b' as string), cast('c' as string))")
-    testFoldConst("SELECT Concat(cast('a' as string), NULL, cast('c' as string))")
-//    testFoldConst("SELECT Concat_Ws(cast('or' as string), cast('d' as string), cast('is' as string))")
-//    testFoldConst("SELECT Concat_Ws(NULL, cast('d' as string), cast('is' as string))")
-//    testFoldConst("SELECT Concat_Ws(cast('or' as string), cast('d' as string), NULL, cast('is' as string))")
-//    testFoldConst("SELECT Concat_Ws(cast('or' as string), cast('d' as string), cast('' as string), cast('is' as string))")
-    testFoldConst("SELECT Ends_With(cast('Hello doris' as string), cast('doris' as string))")
-    testFoldConst("SELECT Ends_With(cast('Hello doris' as string), cast('Hello' as string))")
-    testFoldConst("SELECT Find_In_Set(cast('b' as string), cast('a,b,c' as string))")
-    testFoldConst("SELECT Find_In_Set(cast('d' as string), cast('a,b,c' as string))")
-    testFoldConst("SELECT Find_In_Set(NULL, cast('a,b,c' as string))")
-    testFoldConst("SELECT Find_In_Set(cast('a' as string), NULL)")
-    testFoldConst("SELECT Hex(cast('1' as string))")
-    testFoldConst("SELECT Hex(cast('12' as string))")
-    testFoldConst("SELECT Hex(cast('@' as string))")
-    testFoldConst("SELECT Hex(cast('A' as string))")
-    testFoldConst("SELECT Hex(12)")
-    testFoldConst("SELECT Hex(-1)")
-    testFoldConst("SELECT Hex(cast('hello,doris' as string))")
-    testFoldConst("SELECT Unhex(cast('@' as string))")
-    testFoldConst("SELECT Unhex(cast('68656C6C6F2C646F726973' as string))")
-    testFoldConst("SELECT Unhex(cast('41' as string))")
-    testFoldConst("SELECT Unhex(cast('4142' as string))")
-    testFoldConst("SELECT Unhex(cast('' as string))")
-    testFoldConst("SELECT Unhex(NULL)")
-    testFoldConst("SELECT Instr(cast('abc' as string), cast('b' as string))")
-    testFoldConst("SELECT Instr(cast('abc' as string), cast('d' as string))")
-    testFoldConst("SELECT Instr(cast('abc' as string), NULL)")
-    testFoldConst("SELECT Instr(NULL, cast('a' as string))")
-    testFoldConst("SELECT Lcase(cast('AbC123' as string))")
-    testFoldConst("SELECT Lower(cast('AbC123' as string))")
-    testFoldConst("SELECT Initcap(cast('AbC123abc abc.abc,?|abc' as string))")
-    testFoldConst("SELECT Left(cast('Hello doris' as string), 5)")
-    testFoldConst("SELECT Right(cast('Hello doris' as string), 5)")
-    testFoldConst("SELECT Length(cast('abc' as string))")
-    testFoldConst("SELECT LOCATE(cast('bar' as string), cast('foobarbar' as string))")
-    testFoldConst("SELECT LOCATE(cast('xbar' as string), cast('foobar' as string))")
-    testFoldConst("SELECT LOCATE(cast('' as string), cast('foobar' as string))")
-    testFoldConst("SELECT LOCATE(cast('北京' as string), cast('上海天津北京杭州' as string))")
-    testFoldConst("SELECT Lpad(cast('hi' as string), 5, cast('xy' as string))")
-    testFoldConst("SELECT Lpad(cast('hi' as string), 1, cast('xy' as string))")
-    testFoldConst("SELECT Rpad(cast('hi' as string), 5, cast('xy' as string))")
-    testFoldConst("SELECT Rpad(cast('hi' as string), 1, cast('xy' as string))")
-    testFoldConst("SELECT Ltrim(cast('   ab d' as string))")
-    testFoldConst("SELECT Money_Format(17014116)")
-    testFoldConst("SELECT Money_Format(1123.456)")
-    testFoldConst("SELECT Money_Format(1123.4)")
-    testFoldConst("SELECT Money_Format(Truncate(1000,10))")
-    testFoldConst("SELECT Null_Or_Empty(NULL)")
-    testFoldConst("SELECT Null_Or_Empty(cast('' as string))")
-    testFoldConst("SELECT Null_Or_Empty(cast('a' as string))")
-    testFoldConst("SELECT Not_Null_Or_Empty(NULL)")
-    testFoldConst("SELECT Not_Null_Or_Empty(cast('' as string))")
-    testFoldConst("SELECT Not_Null_Or_Empty(cast('a' as string))")
-    testFoldConst("SELECT Repeat(cast('a' as string), 3)")
-    testFoldConst("SELECT Repeat(cast('a' as string), -1)")
-    testFoldConst("SELECT Repeat(cast('a' as string), 0)")
-    testFoldConst("SELECT Repeat(NULL, 1)")
-    testFoldConst("SELECT Replace(cast('https://doris.apache.org:9090' as string), cast(':9090' as string), cast('' as string))")
-    testFoldConst("SELECT Replace(cast('https://doris.apache.org:9090' as string), cast('' as string), cast('new_str' as string))")
-    testFoldConst("SELECT REVERSE(cast('hello' as string))")
-    testFoldConst("SELECT Split_Part(cast('hello world' as string), cast(' ' as string), 1)")
-    testFoldConst("SELECT Split_Part(cast('hello world' as string), cast(' ' as string), 2)")
-    testFoldConst("SELECT Split_Part(cast('hello world' as string), cast(' ' as string), 3)")
-    testFoldConst("SELECT Concat(CAST('Hello' AS STRING), CAST(' ' AS STRING), CAST('World' AS STRING))")
-    testFoldConst("SELECT Concat(CAST('Hello' AS STRING), CAST(NULL AS STRING))")
-    testFoldConst("SELECT Concat(CAST(NULL AS STRING), CAST('World' AS STRING))")
-
-    testFoldConst("SELECT Starts_With(CAST('hello world' AS STRING), CAST('hello' AS STRING))")
-    testFoldConst("SELECT Starts_With(CAST('hello world' AS STRING), CAST('world' AS STRING))")
-    testFoldConst("SELECT Starts_With(CAST('hello world' AS STRING), CAST(NULL AS STRING))")
-
-    testFoldConst("SELECT StrLeft(CAST(NULL AS STRING), 1)")
-    testFoldConst("SELECT StrLeft(CAST('good morning' AS STRING), NULL)")
-    testFoldConst("SELECT Left(CAST(NULL AS STRING), 1)")
-    testFoldConst("SELECT Left(CAST('good morning' AS STRING), NULL)")
-    testFoldConst("SELECT StrLeft(CAST('Hello doris' AS STRING), 5)")
-    testFoldConst("SELECT Left(CAST('Hello doris' AS STRING), 5)")
-    testFoldConst("SELECT StrRight(CAST(NULL AS STRING), 1)")
-    testFoldConst("SELECT StrRight(CAST('good morning' AS STRING), NULL)")
-    testFoldConst("SELECT Right(CAST(NULL AS STRING), 1)")
-    testFoldConst("SELECT Right(CAST('good morning' AS STRING), NULL)")
-    testFoldConst("SELECT StrRight(CAST('Hello doris' AS STRING), 5)")
-    testFoldConst("SELECT Right(CAST('Hello doris' AS STRING), 5)")
-    testFoldConst("SELECT StrLeft(CAST('good morning' AS STRING), 120)")
-    testFoldConst("SELECT StrLeft(CAST('good morning' AS STRING), -5)")
-    testFoldConst("SELECT StrRight(CAST('Hello doris' AS STRING), 120)")
-    testFoldConst("SELECT StrRight(CAST('Hello doris' AS STRING), -5)")
-    testFoldConst("SELECT Left(CAST('good morning' AS STRING), 120)")
-    testFoldConst("SELECT Left(CAST('good morning' AS STRING), -5)")
-    testFoldConst("SELECT Right(CAST('Hello doris' AS STRING), 120)")
-    testFoldConst("SELECT Right(CAST('Hello doris' AS STRING), -6)")
-
-    testFoldConst("SELECT Substring(CAST('abc1' AS STRING), 2)")
-    testFoldConst("SELECT Substring(CAST('abc1' AS STRING), -2)")
-    testFoldConst("SELECT Substring(CAST('abc1' AS STRING), 5)")
-    testFoldConst("SELECT Substring(CAST('abc1def' AS STRING), 2, 2)")
-    testFoldConst("SELECT Substring(CAST('abcdef' AS STRING), 3, -1)")
-    testFoldConst("SELECT Substring(CAST('abcdef' AS STRING), -3, -1)")
-    testFoldConst("SELECT Substring(CAST('abcdef' AS STRING), 10, 1)")
-
-    testFoldConst("SELECT Substr(CAST('a' AS STRING), 3, 1)")
-    testFoldConst("SELECT Substr(CAST('a' AS STRING), 2, 1)")
-    testFoldConst("SELECT Substr(CAST('a' AS STRING), 1, 1)")
-    testFoldConst("SELECT Substr(CAST('a' AS STRING), 0, 1)")
-    testFoldConst("SELECT Substr(CAST('a' AS STRING), -1, 1)")
-    testFoldConst("SELECT Substr(CAST('a' AS STRING), -2, 1)")
-    testFoldConst("SELECT Substr(CAST('a' AS STRING), -3, 1)")
-    testFoldConst("SELECT Substr(CAST('abcdef' AS STRING), 3, -1)")
-    testFoldConst("SELECT Substr(CAST('abcdef' AS STRING), -3, -1)")
-
-    testFoldConst("SELECT Sub_Replace(CAST('this is origin str' AS STRING), CAST('NEW-STR' AS STRING), 1)")
-    testFoldConst("SELECT Sub_Replace(CAST('doris' AS STRING), CAST('***' AS STRING), 1, 2)")
-
-    testFoldConst("SELECT Substring_Index(CAST('hello world' AS STRING), CAST(' ' AS STRING), 1)")
-    testFoldConst("SELECT Substring_Index(CAST('hello world' AS STRING), CAST(' ' AS STRING), 2)")
-    testFoldConst("SELECT Substring_Index(CAST('hello world' AS STRING), CAST(' ' AS STRING), 3)")
-    testFoldConst("SELECT Substring_Index(CAST('hello world' AS STRING), CAST(' ' AS STRING), -1)")
-    testFoldConst("SELECT Substring_Index(CAST('hello world' AS STRING), CAST(' ' AS STRING), -2)")
-    testFoldConst("SELECT Substring_Index(CAST('hello world' AS STRING), CAST(' ' AS STRING), -3)")
-    testFoldConst("SELECT Substring_Index(CAST('prefix__string2' AS STRING), CAST('__' AS STRING), 2)")
-    testFoldConst("SELECT Substring_Index(CAST('prefix__string2' AS STRING), CAST('_' AS STRING), 2)")
-    testFoldConst("SELECT Substring_Index(CAST('prefix_string2' AS STRING), CAST('__' AS STRING), 1)")
-    testFoldConst("SELECT Substring_Index(CAST(NULL AS STRING), CAST('__' AS STRING), 1)")
-    testFoldConst("SELECT Substring_Index(CAST('prefix_string' AS STRING), CAST(NULL AS STRING), 1)")
-    testFoldConst("SELECT Substring_Index(CAST('prefix_string' AS STRING), CAST('_' AS STRING), NULL)")
-    testFoldConst("SELECT Substring_Index(CAST('prefix_string' AS STRING), CAST('__' AS STRING), -1)")
-
-    testFoldConst("SELECT Elt(0, CAST('hello' AS STRING), CAST('doris' AS STRING))")
-    testFoldConst("SELECT Elt(1, CAST('hello' AS STRING), CAST('doris' AS STRING))")
-    testFoldConst("SELECT Elt(2, CAST('hello' AS STRING), CAST('doris' AS STRING))")
-    testFoldConst("SELECT Elt(3, CAST('hello' AS STRING), CAST('doris' AS STRING))")
-
-    testFoldConst("SELECT Sub_Replace(CAST('this is origin str' AS STRING), CAST('NEW-STR' AS STRING), 1)")
-    testFoldConst("SELECT Sub_Replace(CAST('doris' AS STRING), CAST('***' AS STRING), 1, 2)")
-
-    testFoldConst("SELECT StrCmp(CAST('a' AS STRING), CAST('abc' AS STRING))")
-    testFoldConst("SELECT StrCmp(CAST('abc' AS STRING), CAST('abc' AS STRING))")
-    testFoldConst("SELECT StrCmp(CAST('abcd' AS STRING), CAST('abc' AS STRING))")
-
-    // fix problem of cast date and time function exception
-    testFoldConst("select ifnull(date_format(CONCAT_WS('', '9999-07', '-00'), '%Y-%m'),3)")
-
-    // Normal Usage Test Cases
-
-    // Test Case 1: Append missing trailing character
-    testFoldConst("select append_trailing_char_if_absent('hello', '!')")
-    // Expected Output: 'hello!'
-
-    // Test Case 2: Trailing character already present
-    testFoldConst("select append_trailing_char_if_absent('hello!', '!')")
-    // Expected Output: 'hello!'
-
-    // Test Case 3: Append trailing space
-    testFoldConst("select append_trailing_char_if_absent('hello', ' ')")
-    // Expected Output: 'hello '
-
-    // Test Case 4: Empty string input
-    testFoldConst("select append_trailing_char_if_absent('', '!')")
-    // Expected Output: '!'
-
-    // Test Case 5: Append different character
-    testFoldConst("select append_trailing_char_if_absent('hello', '?')")
-    // Expected Output: 'hello?'
-
-    // Test Case 6: String ends with a different character
-    testFoldConst("select append_trailing_char_if_absent('hello?', '!')")
-    // Expected Output: 'hello?!'
-
-    // Edge and Unusual Usage Test Cases
-
-    // Test Case 7: Input is NULL
-    testFoldConst("select append_trailing_char_if_absent(NULL, '!')")
-    // Expected Output: NULL
-
-    // Test Case 8: Trailing character is NULL
-    testFoldConst("select append_trailing_char_if_absent('hello', NULL)")
-    // Expected Output: NULL
-
-    // Test Case 9: Empty trailing character
-    testFoldConst("select append_trailing_char_if_absent('hello', '')")
-    // Expected Output: Error or no change depending on implementation
-
-    // Test Case 10: Trailing character is more than 1 character long
-    testFoldConst("select append_trailing_char_if_absent('hello', 'ab')")
-    // Expected Output: Error
-
-    // Test Case 11: Input string is a number
-    testFoldConst("select append_trailing_char_if_absent(12345, '!')")
-    // Expected Output: Error or '12345!'
-
-    // Test Case 12: Trailing character is a number
-    testFoldConst("select append_trailing_char_if_absent('hello', '1')")
-    // Expected Output: 'hello1'
-
-    // Test Case 13: Input is a single character
-    testFoldConst("select append_trailing_char_if_absent('h', '!')")
-    // Expected Output: 'h!'
-
-    // Test Case 14: Unicode character as input and trailing character
-    testFoldConst("select append_trailing_char_if_absent('こんにちは', '!')")
-    // Expected Output: 'こんにちは!'
-
-    // Test Case 15: Multibyte character as trailing character
-    testFoldConst("select append_trailing_char_if_absent('hello', '😊')")
-    // Expected Output: 'hello😊'
-
-    // Test Case 16: Long string input
-    testFoldConst("select append_trailing_char_if_absent('This is a very long string', '.')")
-    // Expected Output: 'This is a very long string.'
-
-    // Error Handling Test Cases
-
-    // Test Case 17: Invalid trailing character data type (numeric)
-    testFoldConst("select append_trailing_char_if_absent('hello', 1)")
-    // Expected Output: Error
-
-    // Test Case 18: Invalid input data type (integer)
-    testFoldConst("select append_trailing_char_if_absent(12345, '!')")
-    // Expected Output: Error or '12345!'
-
-    // Test Case 19: Non-ASCII characters
-    testFoldConst("select append_trailing_char_if_absent('Привет', '!')")
-    // Expected Output: 'Привет!'
-
-    // Test Case 20: Trailing character with whitespace
-    testFoldConst("select append_trailing_char_if_absent('hello', ' ')")
-    // Expected Output: 'hello '
-
+    // substring
+    testFoldConst("select substring('1', 1, 1)")
+    testFoldConst("select substring('abc1', -2)")
+    testFoldConst("select substring('abc1', 2)")
+    testFoldConst("select substring('abc1', 5)")
+    testFoldConst("select substring('abc1def', 2, 2)")
+    testFoldConst("select substring('abcdef',10,1)")
+    testFoldConst("select substring('abcdef',-3,-1)")
+    testFoldConst("select substring('abcdef',3,-1)")
+    testFoldConst("select substring(cast('1' as string), 1, 1)")
+    testFoldConst("select substring(CAST('abc1' AS STRING), -2)")
+    testFoldConst("select substring(CAST('abc1' AS STRING), 2)")
+    testFoldConst("select substring(CAST('abc1' AS STRING), 5)")
+    testFoldConst("select substring(CAST('abc1def' AS STRING), 2, 2)")
+    testFoldConst("select substring(CAST('abcdef' AS STRING), 10, 1)")
+    testFoldConst("select substring(CAST('abcdef' AS STRING), -3, -1)")
+    testFoldConst("select substring(CAST('abcdef' AS STRING), 3, -1)")
+    testFoldConst("select substring(cast('Hello' as string), 1, 10)")
+    testFoldConst("select substring(cast('Hello World' as string), -1, 5)")
+    testFoldConst("select substring(cast('Hello World' as string), 1, 5)")
+    testFoldConst("select substring('Hello', 1, 10)")
+    testFoldConst("select substring('Hello World', -1, 5)")
+    testFoldConst("select substring('Hello World', 1, 5)")
+    testFoldConst("select substring('', 1, 5)")
+    testFoldConst("select substring('Hello World', 1, 50)")
+
+    // substring_index
+    testFoldConst("select substring_index('a,b,c', ',', 2)")
+    testFoldConst("select substring_index('a,b,c', '', 2)")
+    testFoldConst("select substring_index(cast('a,b,c' as string), cast(',' as string), 2)")
+    testFoldConst("select substring_index(CAST('hello world' AS STRING), CAST(' ' AS STRING), -1)")
+    testFoldConst("select substring_index(CAST('hello world' AS STRING), CAST(' ' AS STRING), 1)")
+    testFoldConst("select substring_index(CAST('hello world' AS STRING), CAST(' ' AS STRING), -2)")
+    testFoldConst("select substring_index(CAST('hello world' AS STRING), CAST(' ' AS STRING), 2)")
+    testFoldConst("select substring_index(CAST('hello world' AS STRING), CAST(' ' AS STRING), -3)")
+    testFoldConst("select substring_index(CAST('hello world' AS STRING), CAST(' ' AS STRING), 3)")
+    testFoldConst("select substring_index(CAST(NULL AS STRING), CAST('__' AS STRING), 1)")
+    testFoldConst("select substring_index(CAST('prefix_string2' AS STRING), CAST('__' AS STRING), 1)")
+    testFoldConst("select substring_index(CAST('prefix__string2' AS STRING), CAST('_' AS STRING), 2)")
+    testFoldConst("select substring_index(CAST('prefix__string2' AS STRING), CAST('__' AS STRING), 2)")
+    testFoldConst("select substring_index(CAST('prefix_string' AS STRING), CAST('__' AS STRING), -1)")
+    testFoldConst("select substring_index(CAST('prefix_string' AS STRING), CAST('_' AS STRING), NULL)")
+    testFoldConst("select substring_index(CAST('prefix_string' AS STRING), CAST(NULL AS STRING), 1)")
+    testFoldConst("select substring_index('hello world', ' ', -1)")
+    testFoldConst("select substring_index('hello world', ' ', 1)")
+    testFoldConst("select substring_index('hello world', ' ', -2)")
+    testFoldConst("select substring_index('hello world', ' ', 2)")
+    testFoldConst("select substring_index('hello world', ' ', -3)")
+    testFoldConst("select substring_index('hello world', ' ', 3)")
+    testFoldConst("select substring_index(null, '__', 1)")
+    testFoldConst("select substring_index('prefix_string', '__', -1)")
+    testFoldConst("select substring_index('prefix_string2', '__', 1)")
+    testFoldConst("select substring_index('prefix__string2', '_', 2)")
+    testFoldConst("select substring_index('prefix__string2', '__', 2)")
+    testFoldConst("select substring_index('prefix_string', '_', null)")
+    testFoldConst("select substring_index('prefix_string', null, 1)")
+    testFoldConst("SELECT substring_index('哈哈哈AAA','A', -5)")
+    testFoldConst("SELECT substring_index('哈哈哈AAA','A', -4)")
+    testFoldConst("SELECT substring_index('哈哈哈AAA','A', -3)")
+    testFoldConst("SELECT substring_index('哈哈哈AAA','A', -2)")
+    testFoldConst("SELECT substring_index('哈哈哈AAA','A', -1)")
+    testFoldConst("SELECT substring_index('哈哈哈AAA','A', 0)")
+    testFoldConst("SELECT substring_index('哈哈哈AAA','A', 1)")
+    testFoldConst("SELECT substring_index('哈哈哈AAA','A', 2)")
+    testFoldConst("SELECT substring_index('哈哈哈AAA','A', 3)")
+    testFoldConst("SELECT substring_index('哈哈哈AAA','A', 4)")
+    testFoldConst("SELECT substring_index('哈哈哈AAA','A', 5)")
+    testFoldConst("SELECT substring_index('哈哈哈AA+','A', -4)")
+    testFoldConst("SELECT substring_index('哈哈哈AA+','A', -3)")
+    testFoldConst("SELECT substring_index('哈哈哈AA+','A', -2)")
+    testFoldConst("SELECT substring_index('哈哈哈AA+','A', -1)")
+    testFoldConst("SELECT substring_index('哈哈哈AA+','A', 0)")
+    testFoldConst("SELECT substring_index('哈哈哈AA+','A', 1)")
+    testFoldConst("SELECT substring_index('哈哈哈AA+','A', 2)")
+    testFoldConst("SELECT substring_index('哈哈哈AA+','A', 3)")
+    testFoldConst("SELECT substring_index('哈哈哈AA+','A', 4)")
+
+    // trim
+    testFoldConst("select trim('11111', 11)")
+    testFoldConst("select trim(cast('11111' as string), cast(11 as string))")
+    testFoldConst("select trim(cast('  Hello World  ' as string))")
+    testFoldConst("select trim('  Hello World  ')")
+
+    // unhex
+    testFoldConst("select unhex('')")
+    testFoldConst("select unhex('@')")
+    testFoldConst("select unhex('41')")
+    testFoldConst("select unhex('4142')")
+    testFoldConst("select unhex('68656C6C6F2C646F726973')")
+    testFoldConst("select unhex(cast('4142' as string))")
+    testFoldConst("select unhex(cast('41' as string))")
+    testFoldConst("select unhex(cast('68656C6C6F2C646F726973' as string))")
+    testFoldConst("select unhex(cast('' as string))")
+    testFoldConst("select unhex(cast('@' as string))")
+    testFoldConst("select unhex(cast('FF' as string))")
+    testFoldConst("select unhex(cast('GHIJ' as string))")
+    testFoldConst("select unhex('FF')")
+    testFoldConst("select unhex('GHIJ')")
+    testFoldConst("select unhex(NULL)")
+    testFoldConst("select unhex(NULL)")
+    testFoldConst("select upper(cast('Hello World' as string))")
+    testFoldConst("select upper('Hello World')")
+
+    // url_decode url_encode
+    testFoldConst("select url_decode(cast('http%3A%2F%2Fwww.apache.org%2Flicenses%2FLICENSE-2.0' as string))")
+    testFoldConst("select url_decode('http%3A%2F%2Fwww.apache.org%2Flicenses%2FLICENSE-2.0')")
+    testFoldConst("select url_decode('http%3A%2F%2Fwww.apache.org%2Flicenses%2FLICENSE-22.0')")
+    testFoldConst("select url_encode('http://www.apache.org/licenses/LICENSE-2.0')")
+    testFoldConst("select url_encode(' http://www.apache.org/licenses/LICENSE-2.0 ')")
 
 }

From 5725d527b36a97d285188c9e0cad16ed8ff406e2 Mon Sep 17 00:00:00 2001
From: wangbo <wangbo@selectdb.com>
Date: Tue, 7 Jan 2025 11:23:07 +0800
Subject: [PATCH 013/140] Fix threadpool blocked when create thread failed in
 init() (#46366)

---
 be/src/util/threadpool.cpp | 49 +++++++++++++++++---------------------
 be/src/util/threadpool.h   |  3 +++
 2 files changed, 25 insertions(+), 27 deletions(-)

diff --git a/be/src/util/threadpool.cpp b/be/src/util/threadpool.cpp
index 5548ad7f400fdd5..cd87e6e2ae005f7 100644
--- a/be/src/util/threadpool.cpp
+++ b/be/src/util/threadpool.cpp
@@ -273,19 +273,32 @@ ThreadPool::~ThreadPool() {
     shutdown();
 }
 
+Status ThreadPool::try_create_thread(int thread_num, std::lock_guard<std::mutex>&) {
+    for (int i = 0; i < thread_num; i++) {
+        Status status = create_thread();
+        if (status.ok()) {
+            _num_threads_pending_start++;
+        } else {
+            LOG(WARNING) << "Thread pool " << _name << " failed to create thread: " << status;
+            return status;
+        }
+    }
+    return Status::OK();
+}
+
 Status ThreadPool::init() {
     if (!_pool_status.is<UNINITIALIZED>()) {
         return Status::NotSupported("The thread pool {} is already initialized", _name);
     }
     _pool_status = Status::OK();
-    _num_threads_pending_start = _min_threads;
-    for (int i = 0; i < _min_threads; i++) {
-        Status status = create_thread();
-        if (!status.ok()) {
-            shutdown();
-            return status;
-        }
+
+    {
+        std::lock_guard<std::mutex> l(_lock);
+        // create thread failed should not cause threadpool init failed,
+        // because thread can be created later such as when submit a task.
+        static_cast<void>(try_create_thread(_min_threads, l));
     }
+
     // _id of thread pool is used to make sure when we create thread pool with same name, we can
     // get different _metric_entity
     // If not, we will have problem when we deregister entity and register hook.
@@ -688,16 +701,7 @@ Status ThreadPool::set_min_threads(int min_threads) {
     _min_threads = min_threads;
     if (min_threads > _num_threads + _num_threads_pending_start) {
         int addition_threads = min_threads - _num_threads - _num_threads_pending_start;
-        _num_threads_pending_start += addition_threads;
-        for (int i = 0; i < addition_threads; i++) {
-            Status status = create_thread();
-            if (!status.ok()) {
-                _num_threads_pending_start--;
-                LOG(WARNING) << "Thread pool " << _name
-                             << " failed to create thread: " << status.to_string();
-                return status;
-            }
-        }
+        RETURN_IF_ERROR(try_create_thread(addition_threads, l));
     }
     return Status::OK();
 }
@@ -713,16 +717,7 @@ Status ThreadPool::set_max_threads(int max_threads) {
     if (_max_threads > _num_threads + _num_threads_pending_start) {
         int addition_threads = _max_threads - _num_threads - _num_threads_pending_start;
         addition_threads = std::min(addition_threads, _total_queued_tasks);
-        _num_threads_pending_start += addition_threads;
-        for (int i = 0; i < addition_threads; i++) {
-            Status status = create_thread();
-            if (!status.ok()) {
-                _num_threads_pending_start--;
-                LOG(WARNING) << "Thread pool " << _name
-                             << " failed to create thread: " << status.to_string();
-                return status;
-            }
-        }
+        RETURN_IF_ERROR(try_create_thread(addition_threads, l));
     }
     return Status::OK();
 }
diff --git a/be/src/util/threadpool.h b/be/src/util/threadpool.h
index 8c89f570a0a3989..8c262fdfb463465 100644
--- a/be/src/util/threadpool.h
+++ b/be/src/util/threadpool.h
@@ -318,6 +318,9 @@ class ThreadPool {
     // Releases token 't' and invalidates it.
     void release_token(ThreadPoolToken* t);
 
+    //NOTE: not thread safe, caller should keep it thread-safe by using lock
+    Status try_create_thread(int thread_num, std::lock_guard<std::mutex>&);
+
     const std::string _name;
     const std::string _workload_group;
     int _min_threads;

From 61bda40db15791cd2de441b303dd96ed0b5bab9e Mon Sep 17 00:00:00 2001
From: shuke <shuke@selectdb.com>
Date: Tue, 7 Jan 2025 11:40:31 +0800
Subject: [PATCH 014/140] [regression-test](fix) ignore e-2000 in compaction
 (#46436)

### What problem does this PR solve?

Issue Number: close #xxx

Related PR: #xxx

Problem Summary:
In compaction test, there are some situation that triggering compaction
will return E-2000. that's by design and ignore this.
---
 regression-test/plugins/plugin_compaction.groovy | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/regression-test/plugins/plugin_compaction.groovy b/regression-test/plugins/plugin_compaction.groovy
index 385292eb1fe86dc..074505470d52114 100644
--- a/regression-test/plugins/plugin_compaction.groovy
+++ b/regression-test/plugins/plugin_compaction.groovy
@@ -106,7 +106,10 @@ Suite.metaClass.trigger_and_wait_compaction = { String table_name, String compac
                 triggered_tablets.add(tablet) // compaction already in queue, treat it as successfully triggered
             } else if (!auto_compaction_disabled) {
                 // ignore the error if auto compaction enabled
-            } else {
+            } else if (trigger_status.status.contains("E-2000")) {
+                // ignore this tablet compaction.
+            }
+            else {
                 throw new Exception("trigger compaction failed, be host: ${be_host}, tablet id: ${tablet.TabletId}, status: ${trigger_status.status}")
             }
         } else {

From 56d1312317e62dab1fadd079a2bad99564435142 Mon Sep 17 00:00:00 2001
From: shuke <shuke@selectdb.com>
Date: Tue, 7 Jan 2025 11:45:20 +0800
Subject: [PATCH 015/140] [regression-test](fix) fix
 test_bloom_filter_hit.groovy case bug (#46438)

### What problem does this PR solve?

Issue Number: close #xxx

Related PR: #xxx

Problem Summary:
This case suppose parallel_fragment_exec_instance_num=1, add to make it
stable.
---
 .../suites/bloom_filter_p0/test_bloom_filter_hit.groovy          | 1 +
 1 file changed, 1 insertion(+)

diff --git a/regression-test/suites/bloom_filter_p0/test_bloom_filter_hit.groovy b/regression-test/suites/bloom_filter_p0/test_bloom_filter_hit.groovy
index f62c02a4205fe65..3c7b73407434f19 100644
--- a/regression-test/suites/bloom_filter_p0/test_bloom_filter_hit.groovy
+++ b/regression-test/suites/bloom_filter_p0/test_bloom_filter_hit.groovy
@@ -129,6 +129,7 @@ suite("test_bloom_filter_hit") {
     sql """ set parallel_scan_min_rows_per_scanner = 2097152; """
 
     // bf filter
+    sql """ SET parallel_pipeline_task_num=1 """
     qt_sql """ select * from test_ip_bf where ip_v6='4a3e:dc26:1819:83e6:9ee5:7239:ff44:aee8' """
     profiles = httpGet(profileUrl)
     log.debug("profiles:{}", profiles);

From 131742d492d9987c20d566007e4c9eb5db1958f5 Mon Sep 17 00:00:00 2001
From: wuwenchi <wuwenchi@selectdb.com>
Date: Tue, 7 Jan 2025 12:01:09 +0800
Subject: [PATCH 016/140] [fix](hive) Incorrect location conversion (#46362)

### What problem does this PR solve?

Problem Summary:

For the "original_write_path" field, when the location is of the S3
type, it should not be converted into the S3 schema. Instead, the
original location should be retained.
---
 .../apache/doris/planner/HiveTableSink.java   |   2 +-
 .../doris/planner/HiveTableSinkTest.java      | 145 ++++++++++++++++++
 2 files changed, 146 insertions(+), 1 deletion(-)
 create mode 100644 fe/fe-core/src/test/java/org/apache/doris/planner/HiveTableSinkTest.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HiveTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HiveTableSink.java
index 168f92c113cbe40..3635d10633f108b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/HiveTableSink.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HiveTableSink.java
@@ -128,7 +128,7 @@ public void bindDataSink(Optional<InsertCommandContext> insertCtx)
         TFileType fileType = locationPath.getTFileTypeForBE();
         if (fileType == TFileType.FILE_S3) {
             locationParams.setWritePath(storageLocation);
-            locationParams.setOriginalWritePath(location);
+            locationParams.setOriginalWritePath(sd.getLocation());
             locationParams.setTargetPath(location);
             if (insertCtx.isPresent()) {
                 HiveInsertCommandContext context = (HiveInsertCommandContext) insertCtx.get();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/HiveTableSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/HiveTableSinkTest.java
new file mode 100644
index 000000000000000..8794a56eac91388
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/planner/HiveTableSinkTest.java
@@ -0,0 +1,145 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.planner;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.datasource.hive.HMSCachedClient;
+import org.apache.doris.datasource.hive.HMSExternalCatalog;
+import org.apache.doris.datasource.hive.HMSExternalDatabase;
+import org.apache.doris.datasource.hive.HMSExternalTable;
+import org.apache.doris.datasource.hive.ThriftHMSCachedClient;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.junit.Test;
+import org.locationtech.jts.util.Assert;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+public class HiveTableSinkTest {
+
+    @Test
+    public void testBindDataSink() throws AnalysisException {
+
+        new MockUp<ThriftHMSCachedClient>() {
+            @Mock
+            List<Partition> listPartitions(String dbName, String tblName) {
+                return new ArrayList<Partition>() {{
+                        add(new Partition() {{
+                                setValues(new ArrayList<String>() {{
+                                        add("a");
+                                    }
+                                });
+                                setSd(new StorageDescriptor() {{
+                                        setInputFormat("orc");
+                                    }
+                                });
+                            }
+                        });
+                    }
+                };
+            }
+        };
+
+        new MockUp<HMSExternalCatalog>() {
+            @Mock
+            public HMSCachedClient getClient() {
+                return new ThriftHMSCachedClient(null, 2);
+            }
+        };
+
+        ArrayList<String> locations = new ArrayList<String>() {{
+                add("gs://abc/def");
+                add("s3://abc/def");
+                add("s3a://abc/def");
+                add("s3n://abc/def");
+                add("bos://abc/def");
+                add("oss://abc/def");
+                add("cos://abc/def");
+            }
+        };
+        for (String location : locations) {
+            mockDifferLocationTable(location);
+
+            HMSExternalCatalog hmsExternalCatalog = new HMSExternalCatalog();
+            hmsExternalCatalog.setInitialized(true);
+            HMSExternalDatabase db = new HMSExternalDatabase(hmsExternalCatalog, 10000, "hive_db1", "hive_db1");
+            HMSExternalTable tbl = new HMSExternalTable(10001, "hive_tbl1", "hive_db1", hmsExternalCatalog, db);
+            HiveTableSink hiveTableSink = new HiveTableSink(tbl);
+            hiveTableSink.bindDataSink(Optional.empty());
+
+            Assert.equals(hiveTableSink.tDataSink.hive_table_sink.location.original_write_path, location);
+        }
+    }
+
+    private void mockDifferLocationTable(String location) {
+        new MockUp<HMSExternalTable>() {
+            @Mock
+            public Set<String> getPartitionColumnNames() {
+                return new HashSet<String>() {{
+                        add("a");
+                        add("b");
+                    }
+                };
+            }
+
+            @Mock
+            public List<Column> getColumns() {
+                Column a = new Column("a", PrimitiveType.INT);
+                Column b = new Column("b", PrimitiveType.INT);
+                return new ArrayList<Column>() {{
+                        add(a);
+                        add(b);
+                    }
+                };
+            }
+
+            @Mock
+            public org.apache.hadoop.hive.metastore.api.Table getRemoteTable() {
+                Table table = new Table();
+                table.setSd(new StorageDescriptor() {{
+                        setInputFormat("orc");
+                        setBucketCols(new ArrayList<>());
+                        setNumBuckets(1);
+                        setSerdeInfo(new SerDeInfo() {{
+                                setParameters(new HashMap<>());
+                            }
+                        });
+                        setLocation(location);
+                    }
+                });
+                table.setParameters(new HashMap<String, String>() {{
+                        put("orc.compress", "lzo");
+                    }
+                });
+                return table;
+            }
+        };
+    }
+}

From 04ef8d8812b6e49db018baa71ee59052fc4db4b6 Mon Sep 17 00:00:00 2001
From: LiBinfeng <libinfeng@selectdb.com>
Date: Tue, 7 Jan 2025 13:20:02 +0800
Subject: [PATCH 017/140] [fix](cases) fix unstable leading cases (#46441)

### What problem does this PR solve?

Issue Number: close #xxx

Related PR: #xxx

Problem Summary:

test_leading case unstable because do not explicit assign all distribute
type for all joins, solved by add all distribute type for all joins
---
 .../data/nereids_p0/hint/test_leading.out     | 118 ++++++------------
 .../nereids_p0/hint/test_leading.groovy       |  32 ++---
 2 files changed, 52 insertions(+), 98 deletions(-)

diff --git a/regression-test/data/nereids_p0/hint/test_leading.out b/regression-test/data/nereids_p0/hint/test_leading.out
index 7b0c9c4bcb05006..ea6254496366885 100644
--- a/regression-test/data/nereids_p0/hint/test_leading.out
+++ b/regression-test/data/nereids_p0/hint/test_leading.out
@@ -2147,52 +2147,6 @@
 -- !select88_13 --
 119
 
--- !select90_1 --
-PhysicalResultSink
---hashAgg[GLOBAL]
-----PhysicalDistribute[DistributionSpecGather]
-------hashAgg[LOCAL]
---------hashJoin[INNER_JOIN broadcast] hashCondition=((t1.c1 = t2.c2)) otherCondition=()
-----------PhysicalOlapScan[t1]
-----------PhysicalOlapScan[t2]
-
-Hint log:
-Used: [broadcast]_2
-UnUsed:
-SyntaxError:
-
--- !select90_3 --
-PhysicalResultSink
---hashAgg[GLOBAL]
-----PhysicalDistribute[DistributionSpecGather]
-------hashAgg[LOCAL]
---------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t2.c2 = t3.c3)) otherCondition=()
-----------hashJoin[INNER_JOIN broadcast] hashCondition=((t1.c1 = t2.c2)) otherCondition=()
-------------PhysicalOlapScan[t1]
-------------PhysicalOlapScan[t2]
-----------PhysicalOlapScan[t3]
-
-Hint log:
-Used: [broadcast]_2 [shuffle]_3
-UnUsed:
-SyntaxError:
-
--- !select90_5 --
-PhysicalResultSink
---hashAgg[GLOBAL]
-----PhysicalDistribute[DistributionSpecGather]
-------hashAgg[LOCAL]
---------hashJoin[RIGHT_OUTER_JOIN shuffle] hashCondition=((t2.c2 = t3.c3)) otherCondition=()
-----------hashJoin[INNER_JOIN broadcast] hashCondition=((t1.c1 = t2.c2)) otherCondition=()
-------------PhysicalOlapScan[t1]
-------------PhysicalOlapScan[t2]
-----------PhysicalOlapScan[t3]
-
-Hint log:
-Used: [broadcast]_2 [shuffle]_3
-UnUsed:
-SyntaxError:
-
 -- !select92_1 --
 PhysicalResultSink
 --hashAgg[GLOBAL]
@@ -2301,8 +2255,8 @@ PhysicalResultSink
 ----------PhysicalOlapScan[t3]
 
 Hint log:
-Used: leading(t1 t2 broadcast t3 ) 
-UnUsed: 
+Used: leading(t1 broadcast t2 broadcast t3 ) 
+UnUsed:
 SyntaxError:
 
 -- !select93_2 --
@@ -2317,8 +2271,8 @@ PhysicalResultSink
 ------------PhysicalOlapScan[t3]
 
 Hint log:
-Used: leading(t1 { t2 broadcast t3 } ) 
-UnUsed: 
+Used: leading(t1 broadcast { t2 broadcast t3 } ) 
+UnUsed:
 SyntaxError:
 
 -- !select93_3 --
@@ -2333,8 +2287,8 @@ PhysicalResultSink
 ------------PhysicalOlapScan[t2]
 
 Hint log:
-Used: leading(t1 { t3 broadcast t2 } ) 
-UnUsed: 
+Used: leading(t1 broadcast { t3 broadcast t2 } ) 
+UnUsed:
 SyntaxError:
 
 -- !select93_4 --
@@ -2349,8 +2303,8 @@ PhysicalResultSink
 ----------PhysicalOlapScan[t3]
 
 Hint log:
-Used: leading(t2 t1 broadcast t3 ) 
-UnUsed: 
+Used: leading(t2 broadcast t1 broadcast t3 ) 
+UnUsed:
 SyntaxError:
 
 -- !select93_5 --
@@ -2365,8 +2319,8 @@ PhysicalResultSink
 ------------PhysicalOlapScan[t3]
 
 Hint log:
-Used: leading(t2 { t1 broadcast t3 } ) 
-UnUsed: 
+Used: leading(t2 broadcast { t1 broadcast t3 } ) 
+UnUsed:
 SyntaxError:
 
 -- !select93_6 --
@@ -2381,8 +2335,8 @@ PhysicalResultSink
 ------------PhysicalOlapScan[t1]
 
 Hint log:
-Used: leading(t2 { t3 broadcast t1 } ) 
-UnUsed: 
+Used: leading(t2 broadcast { t3 broadcast t1 } ) 
+UnUsed:
 SyntaxError:
 
 -- !select94_2 --
@@ -2390,15 +2344,15 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----PhysicalDistribute[DistributionSpecGather]
 ------hashAgg[LOCAL]
---------hashJoin[INNER_JOIN broadcast] hashCondition=((t2.c2 = t3.c3)) otherCondition=()
+--------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t2.c2 = t3.c3)) otherCondition=()
 ----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t1.c1 = t2.c2)) otherCondition=()
 ------------PhysicalOlapScan[t1]
 ------------PhysicalOlapScan[t2]
 ----------PhysicalOlapScan[t3]
 
 Hint log:
-Used: leading(t1 shuffle t2 t3 ) 
-UnUsed: 
+Used: leading(t1 shuffle t2 shuffle t3 ) 
+UnUsed:
 SyntaxError:
 
 -- !select94_2 --
@@ -2408,13 +2362,13 @@ PhysicalResultSink
 ------hashAgg[LOCAL]
 --------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t1.c1 = t2.c2)) otherCondition=()
 ----------PhysicalOlapScan[t1]
-----------hashJoin[INNER_JOIN broadcast] hashCondition=((t2.c2 = t3.c3)) otherCondition=()
+----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t2.c2 = t3.c3)) otherCondition=()
 ------------PhysicalOlapScan[t2]
 ------------PhysicalOlapScan[t3]
 
 Hint log:
-Used: leading(t1 shuffle { t2 t3 } ) 
-UnUsed: 
+Used: leading(t1 shuffle { t2 shuffle t3 } ) 
+UnUsed:
 SyntaxError:
 
 -- !select94_2 --
@@ -2424,13 +2378,13 @@ PhysicalResultSink
 ------hashAgg[LOCAL]
 --------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t1.c1 = t2.c2)) otherCondition=()
 ----------PhysicalOlapScan[t1]
-----------hashJoin[INNER_JOIN broadcast] hashCondition=((t2.c2 = t3.c3)) otherCondition=()
+----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t2.c2 = t3.c3)) otherCondition=()
 ------------PhysicalOlapScan[t3]
 ------------PhysicalOlapScan[t2]
 
 Hint log:
-Used: leading(t1 shuffle { t3 t2 } ) 
-UnUsed: 
+Used: leading(t1 shuffle { t3 shuffle t2 } ) 
+UnUsed:
 SyntaxError:
 
 -- !select94_2 --
@@ -2438,15 +2392,15 @@ PhysicalResultSink
 --hashAgg[GLOBAL]
 ----PhysicalDistribute[DistributionSpecGather]
 ------hashAgg[LOCAL]
---------hashJoin[INNER_JOIN broadcast] hashCondition=((t2.c2 = t3.c3)) otherCondition=()
+--------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t2.c2 = t3.c3)) otherCondition=()
 ----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t1.c1 = t2.c2)) otherCondition=()
 ------------PhysicalOlapScan[t2]
 ------------PhysicalOlapScan[t1]
 ----------PhysicalOlapScan[t3]
 
 Hint log:
-Used: leading(t2 shuffle t1 t3 ) 
-UnUsed: 
+Used: leading(t2 shuffle t1 shuffle t3 ) 
+UnUsed:
 SyntaxError:
 
 -- !select94_2 --
@@ -2456,13 +2410,13 @@ PhysicalResultSink
 ------hashAgg[LOCAL]
 --------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t1.c1 = t2.c2) and (t2.c2 = t3.c3)) otherCondition=()
 ----------PhysicalOlapScan[t2]
-----------hashJoin[INNER_JOIN broadcast] hashCondition=((t1.c1 = t3.c3)) otherCondition=()
+----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t1.c1 = t3.c3)) otherCondition=()
 ------------PhysicalOlapScan[t1]
 ------------PhysicalOlapScan[t3]
 
 Hint log:
-Used: leading(t2 shuffle { t1 t3 } ) 
-UnUsed: 
+Used: leading(t2 shuffle { t1 shuffle t3 } ) 
+UnUsed:
 SyntaxError:
 
 -- !select94_2 --
@@ -2472,13 +2426,13 @@ PhysicalResultSink
 ------hashAgg[LOCAL]
 --------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t1.c1 = t2.c2) and (t2.c2 = t3.c3)) otherCondition=()
 ----------PhysicalOlapScan[t2]
-----------hashJoin[INNER_JOIN broadcast] hashCondition=((t1.c1 = t3.c3)) otherCondition=()
+----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t1.c1 = t3.c3)) otherCondition=()
 ------------PhysicalOlapScan[t3]
 ------------PhysicalOlapScan[t1]
 
 Hint log:
-Used: leading(t2 shuffle { t3 t1 } ) 
-UnUsed: 
+Used: leading(t2 shuffle { t3 shuffle t1 } ) 
+UnUsed:
 SyntaxError:
 
 -- !select95_1 --
@@ -2509,8 +2463,8 @@ PhysicalResultSink
 ----------PhysicalOlapScan[t3]
 
 Hint log:
-Used: leading(t2 broadcast t1 t3 ) 
-UnUsed: 
+Used: leading(t2 broadcast t1 broadcast t3 ) 
+UnUsed:
 SyntaxError:
 
 -- !select95_8 --
@@ -2525,8 +2479,8 @@ PhysicalResultSink
 ------------PhysicalOlapScan[t2]
 
 Hint log:
-Used: leading(t3 broadcast { t1 t2 } ) 
-UnUsed: 
+Used: leading(t3 broadcast { t1 broadcast t2 } ) 
+UnUsed:
 SyntaxError:
 
 -- !select95_9 --
@@ -2541,8 +2495,8 @@ PhysicalResultSink
 ------------PhysicalOlapScan[t1]
 
 Hint log:
-Used: leading(t3 broadcast { t2 t1 } ) 
-UnUsed: 
+Used: leading(t3 broadcast { t2 broadcast t1 } ) 
+UnUsed:
 SyntaxError:
 
 -- !select96_1 --
diff --git a/regression-test/suites/nereids_p0/hint/test_leading.groovy b/regression-test/suites/nereids_p0/hint/test_leading.groovy
index 5e79cc6f25360f1..8a4e27ea7fb5401 100644
--- a/regression-test/suites/nereids_p0/hint/test_leading.groovy
+++ b/regression-test/suites/nereids_p0/hint/test_leading.groovy
@@ -982,19 +982,19 @@ suite("test_leading") {
     qt_select92_5 """explain shape plan select /*+ leading(t2 shuffle {t1 broadcast t3}) */ count(*) from t1 join t2 on c1 = c2 join t3 on c2 = c3;"""
     qt_select92_6 """explain shape plan select /*+ leading(t2 shuffle {t3 broadcast t1}) */ count(*) from t1 join t2 on c1 = c2 join t3 on c2 = c3;"""
 
-    qt_select93_1 """explain shape plan select /*+ leading(t1 t2 broadcast t3) */ count(*) from t1 join t2 on c1 = c2 join t3 on c2 = c3;"""
-    qt_select93_2 """explain shape plan select /*+ leading(t1 {t2 broadcast t3}) */ count(*) from t1 join t2 on c1 = c2 join t3 on c2 = c3;"""
-    qt_select93_3 """explain shape plan select /*+ leading(t1 {t3 broadcast t2}) */ count(*) from t1 join t2 on c1 = c2 join t3 on c2 = c3;"""
-    qt_select93_4 """explain shape plan select /*+ leading(t2 t1 broadcast t3) */ count(*) from t1 join t2 on c1 = c2 join t3 on c2 = c3;"""
-    qt_select93_5 """explain shape plan select /*+ leading(t2 {t1 broadcast t3}) */ count(*) from t1 join t2 on c1 = c2 join t3 on c2 = c3;"""
-    qt_select93_6 """explain shape plan select /*+ leading(t2 {t3 broadcast t1}) */ count(*) from t1 join t2 on c1 = c2 join t3 on c2 = c3;"""
-
-    qt_select94_2 """explain shape plan select /*+ leading(t1 shuffle t2 t3) */ count(*) from t1 join t2 on c1 = c2 join t3 on c2 = c3;"""
-    qt_select94_2 """explain shape plan select /*+ leading(t1 shuffle {t2 t3}) */ count(*) from t1 join t2 on c1 = c2 join t3 on c2 = c3;"""
-    qt_select94_2 """explain shape plan select /*+ leading(t1 shuffle {t3 t2}) */ count(*) from t1 join t2 on c1 = c2 join t3 on c2 = c3;"""
-    qt_select94_2 """explain shape plan select /*+ leading(t2 shuffle t1 t3) */ count(*) from t1 join t2 on c1 = c2 join t3 on c2 = c3;"""
-    qt_select94_2 """explain shape plan select /*+ leading(t2 shuffle {t1 t3}) */ count(*) from t1 join t2 on c1 = c2 join t3 on c2 = c3;"""
-    qt_select94_2 """explain shape plan select /*+ leading(t2 shuffle {t3 t1}) */ count(*) from t1 join t2 on c1 = c2 join t3 on c2 = c3;"""
+    qt_select93_1 """explain shape plan select /*+ leading(t1 broadcast t2 broadcast t3) */ count(*) from t1 join t2 on c1 = c2 join t3 on c2 = c3;"""
+    qt_select93_2 """explain shape plan select /*+ leading(t1 broadcast {t2 broadcast t3}) */ count(*) from t1 join t2 on c1 = c2 join t3 on c2 = c3;"""
+    qt_select93_3 """explain shape plan select /*+ leading(t1 broadcast {t3 broadcast t2}) */ count(*) from t1 join t2 on c1 = c2 join t3 on c2 = c3;"""
+    qt_select93_4 """explain shape plan select /*+ leading(t2 broadcast t1 broadcast t3) */ count(*) from t1 join t2 on c1 = c2 join t3 on c2 = c3;"""
+    qt_select93_5 """explain shape plan select /*+ leading(t2 broadcast {t1 broadcast t3}) */ count(*) from t1 join t2 on c1 = c2 join t3 on c2 = c3;"""
+    qt_select93_6 """explain shape plan select /*+ leading(t2 broadcast {t3 broadcast t1}) */ count(*) from t1 join t2 on c1 = c2 join t3 on c2 = c3;"""
+
+    qt_select94_2 """explain shape plan select /*+ leading(t1 shuffle t2 shuffle t3) */ count(*) from t1 join t2 on c1 = c2 join t3 on c2 = c3;"""
+    qt_select94_2 """explain shape plan select /*+ leading(t1 shuffle {t2 shuffle t3}) */ count(*) from t1 join t2 on c1 = c2 join t3 on c2 = c3;"""
+    qt_select94_2 """explain shape plan select /*+ leading(t1 shuffle {t3 shuffle t2}) */ count(*) from t1 join t2 on c1 = c2 join t3 on c2 = c3;"""
+    qt_select94_2 """explain shape plan select /*+ leading(t2 shuffle t1 shuffle t3) */ count(*) from t1 join t2 on c1 = c2 join t3 on c2 = c3;"""
+    qt_select94_2 """explain shape plan select /*+ leading(t2 shuffle {t1 shuffle t3}) */ count(*) from t1 join t2 on c1 = c2 join t3 on c2 = c3;"""
+    qt_select94_2 """explain shape plan select /*+ leading(t2 shuffle {t3 shuffle t1}) */ count(*) from t1 join t2 on c1 = c2 join t3 on c2 = c3;"""
 
     // outer join
     qt_select95_1 """explain shape plan select /*+ leading(t1 broadcast t2 broadcast t3) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;"""
@@ -1006,7 +1006,7 @@ suite("test_leading") {
         sql """shape plan select /*+ leading(t1 broadcast {t3 t2}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;"""
         contains("UnUsed: leading(t1 broadcast { t3 t2 })")
     }
-    qt_select95_4 """explain shape plan select /*+ leading(t2 broadcast t1 t3) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;"""
+    qt_select95_4 """explain shape plan select /*+ leading(t2 broadcast t1 broadcast t3) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;"""
     explain {
         sql """shape plan select /*+ leading(t2 broadcast {t1 t3}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;"""
         contains("UnUsed: leading(t2 broadcast { t1 t3 })")
@@ -1019,8 +1019,8 @@ suite("test_leading") {
         sql """shape plan select /*+ leading(t3 broadcast t1 t2) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;"""
         contains("UnUsed: leading(t3 broadcast t1 t2)")
     }
-    qt_select95_8 """explain shape plan select /*+ leading(t3 broadcast {t1 t2}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;"""
-    qt_select95_9 """explain shape plan select /*+ leading(t3 broadcast {t2 t1}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;"""
+    qt_select95_8 """explain shape plan select /*+ leading(t3 broadcast {t1 broadcast t2}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;"""
+    qt_select95_9 """explain shape plan select /*+ leading(t3 broadcast {t2 broadcast t1}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;"""
 
     qt_select96_1 """explain shape plan select /*+ leading(t1 shuffle t2 broadcast t3) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;"""
     explain {

From c5fce7a7a65e4b9d91c3272612d161f736c7d7d1 Mon Sep 17 00:00:00 2001
From: shuke <shuke@selectdb.com>
Date: Tue, 7 Jan 2025 14:12:09 +0800
Subject: [PATCH 018/140] [regression-test](fix) fix case bug in alter table 
 (#46435)

### What problem does this PR solve?

Issue Number: close #xxx

Related PR: #xxx
#44058

Problem Summary:
behavior change in #44058
---
 .../suites/unique_with_mow_c_p0/test_schema_change_ck.groovy    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/regression-test/suites/unique_with_mow_c_p0/test_schema_change_ck.groovy b/regression-test/suites/unique_with_mow_c_p0/test_schema_change_ck.groovy
index fb59aabf05dcd10..fa7ca91d88aa377 100644
--- a/regression-test/suites/unique_with_mow_c_p0/test_schema_change_ck.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/test_schema_change_ck.groovy
@@ -117,7 +117,7 @@ suite("test_schema_change_ck") {
     /****** drop key column ******/
     test {
         sql """ alter table ${tableName} drop column k2; """
-        exception "Can not drop key column in Unique data model table"
+        exception "Can not drop key column k2 in unique data model table"
     }
 
     /****** TODO does not support drop cluster key ******/

From ac0322dc7461df147367a20ed6499802bcd5c387 Mon Sep 17 00:00:00 2001
From: Pxl <xl@selectdb.com>
Date: Tue, 7 Jan 2025 14:56:09 +0800
Subject: [PATCH 019/140] [Enchancement](sort) change priority_queue to ck
 SortingQueue (#45952)

1. change priority_queue to ck SortingQueue(a heap whitch support modify
top element)
2. avoid some convert_if_const usage


![QQ_1735532950407](https://github.com/user-attachments/assets/ec7b52c1-424b-4d7f-993c-64410ec35ba5)
---
 be/src/vec/columns/column.h                   |   1 -
 be/src/vec/common/pod_array.h                 |   8 +-
 be/src/vec/common/sort/partition_sorter.cpp   | 197 ++++++------
 be/src/vec/common/sort/partition_sorter.h     |  19 +-
 be/src/vec/common/sort/sorter.cpp             | 107 ++++---
 be/src/vec/common/sort/sorter.h               |  37 +--
 be/src/vec/core/sort_cursor.h                 | 280 ++++++++++++++++--
 .../doris/planner/PartitionSortNode.java      |   5 +-
 8 files changed, 460 insertions(+), 194 deletions(-)

diff --git a/be/src/vec/columns/column.h b/be/src/vec/columns/column.h
index 729e5470c97ad58..bc8ab101ecf9001 100644
--- a/be/src/vec/columns/column.h
+++ b/be/src/vec/columns/column.h
@@ -196,7 +196,6 @@ class IColumn : public COW<IColumn> {
 
     /// Appends range of elements from other column with the same type.
     /// Could be used to concatenate columns.
-    /// TODO: we need `insert_range_from_const` for every column type.
     virtual void insert_range_from(const IColumn& src, size_t start, size_t length) = 0;
 
     /// Appends range of elements from other column with the same type.
diff --git a/be/src/vec/common/pod_array.h b/be/src/vec/common/pod_array.h
index f798ca69bd68ddd..c55dc0cc33eee30 100644
--- a/be/src/vec/common/pod_array.h
+++ b/be/src/vec/common/pod_array.h
@@ -355,14 +355,14 @@ class PODArray : public PODArrayBase<sizeof(T), initial_bytes, TAllocator, pad_r
     /// The index is signed to access -1th element without pointer overflow.
     T& operator[](ssize_t n) {
         /// <= size, because taking address of one element past memory range is Ok in C++ (expression like &arr[arr.size()] is perfectly valid).
-        assert((n >= (static_cast<ssize_t>(pad_left_) ? -1 : 0)) &&
-               (n <= static_cast<ssize_t>(this->size())));
+        DCHECK_GE(n, (static_cast<ssize_t>(pad_left_) ? -1 : 0));
+        DCHECK_LE(n, static_cast<ssize_t>(this->size()));
         return t_start()[n];
     }
 
     const T& operator[](ssize_t n) const {
-        assert((n >= (static_cast<ssize_t>(pad_left_) ? -1 : 0)) &&
-               (n <= static_cast<ssize_t>(this->size())));
+        DCHECK_GE(n, (static_cast<ssize_t>(pad_left_) ? -1 : 0));
+        DCHECK_LE(n, static_cast<ssize_t>(this->size()));
         return t_start()[n];
     }
 
diff --git a/be/src/vec/common/sort/partition_sorter.cpp b/be/src/vec/common/sort/partition_sorter.cpp
index c47a8a8895298ef..f6a3cb443aae77e 100644
--- a/be/src/vec/common/sort/partition_sorter.cpp
+++ b/be/src/vec/common/sort/partition_sorter.cpp
@@ -49,9 +49,11 @@ PartitionSorter::PartitionSorter(VSortExecExprs& vsort_exec_exprs, int limit, in
         : Sorter(vsort_exec_exprs, limit, offset, pool, is_asc_order, nulls_first),
           _state(MergeSorterState::create_unique(row_desc, offset, limit, state, profile)),
           _row_desc(row_desc),
-          _has_global_limit(has_global_limit),
           _partition_inner_limit(partition_inner_limit),
-          _top_n_algorithm(top_n_algorithm),
+          _top_n_algorithm(
+                  has_global_limit
+                          ? TopNAlgorithm::ROW_NUMBER
+                          : top_n_algorithm), // FE will make this modification, but still maintain this code for compatibility
           _previous_row(previous_row) {}
 
 Status PartitionSorter::append_block(Block* input_block) {
@@ -64,10 +66,13 @@ Status PartitionSorter::append_block(Block* input_block) {
 
 Status PartitionSorter::prepare_for_read() {
     auto& blocks = _state->get_sorted_block();
-    auto& priority_queue = _state->get_priority_queue();
+    auto& queue = _state->get_queue();
+    std::vector<MergeSortCursor> cursors;
     for (auto& block : blocks) {
-        priority_queue.emplace(MergeSortCursorImpl::create_shared(block, _sort_description));
+        cursors.emplace_back(
+                MergeSortCursorImpl::create_shared(std::move(block), _sort_description));
     }
+    queue = MergeSorterQueue(cursors);
     blocks.clear();
     return Status::OK();
 }
@@ -88,122 +93,114 @@ void PartitionSorter::reset_sorter_state(RuntimeState* runtime_state) {
 }
 
 Status PartitionSorter::get_next(RuntimeState* state, Block* block, bool* eos) {
-    if (_state->get_priority_queue().empty()) {
-        *eos = true;
-    } else if (_state->get_priority_queue().size() == 1 && _has_global_limit) {
-        block->swap(*_state->get_priority_queue().top().impl->block);
-        block->set_num_rows(_partition_inner_limit);
-        *eos = true;
+    if (_top_n_algorithm == TopNAlgorithm::ROW_NUMBER) {
+        return _read_row_num(block, eos, state->batch_size());
     } else {
-        RETURN_IF_ERROR(partition_sort_read(block, eos, state->batch_size()));
+        return _read_row_rank(block, eos, state->batch_size());
     }
-    return Status::OK();
 }
 
-Status PartitionSorter::partition_sort_read(Block* output_block, bool* eos, int batch_size) {
-    auto& priority_queue = _state->get_priority_queue();
-    const auto& sorted_block = priority_queue.top().impl->block;
-    size_t num_columns = sorted_block->columns();
+Status PartitionSorter::_read_row_num(Block* output_block, bool* eos, int batch_size) {
+    auto& queue = _state->get_queue();
+    size_t num_columns = _state->unsorted_block()->columns();
+
     MutableBlock m_block =
-            VectorizedUtils::build_mutable_mem_reuse_block(output_block, *sorted_block);
+            VectorizedUtils::build_mutable_mem_reuse_block(output_block, *_state->unsorted_block());
     MutableColumns& merged_columns = m_block.mutable_columns();
-    size_t current_output_rows = 0;
-
-    bool get_enough_data = false;
-    while (!priority_queue.empty()) {
-        auto current = priority_queue.top();
-        priority_queue.pop();
-        if (UNLIKELY(_previous_row->impl == nullptr)) {
-            *_previous_row = current;
+    size_t merged_rows = 0;
+
+    Defer defer {[&]() {
+        if (merged_rows == 0 || _get_enough_data()) {
+            *eos = true;
         }
+    }};
 
-        switch (_top_n_algorithm) {
-        case TopNAlgorithm::ROW_NUMBER: {
-            //1 row_number no need to check distinct, just output partition_inner_limit row
-            if ((current_output_rows + _output_total_rows) < _partition_inner_limit) {
-                for (size_t i = 0; i < num_columns; ++i) {
-                    merged_columns[i]->insert_from(*current->block->get_columns()[i], current->pos);
-                }
-            } else {
-                //rows has get enough
-                get_enough_data = true;
+    while (queue.is_valid() && merged_rows < batch_size && !_get_enough_data()) {
+        auto [current, current_rows] = queue.current();
+
+        // row_number no need to check distinct, just output partition_inner_limit row
+        size_t needed_rows = _partition_inner_limit - _output_total_rows;
+        size_t step = std::min(needed_rows, std::min(current_rows, batch_size - merged_rows));
+
+        if (current->impl->is_last(step) && current->impl->pos == 0) {
+            if (merged_rows != 0) {
+                // return directly for next time's read swap whole block
+                return Status::OK();
             }
-            current_output_rows++;
-            break;
+            // swap and return block directly when we should get all data from cursor
+            output_block->swap(*current->impl->block);
+            merged_rows += step;
+            _output_total_rows += step;
+            queue.remove_top();
+            return Status::OK();
         }
-        case TopNAlgorithm::DENSE_RANK: {
-            //  dense_rank(): 1,1,1,2,2,2,2,.......,2,3,3,3, if SQL: where rk < 3, need output all 1 and 2
-            //3 dense_rank() maybe need distinct rows of partition_inner_limit
-            //3.1 _has_global_limit = true, so check (current_output_rows + _output_total_rows) >= _partition_inner_limit)
-            //3.2 _has_global_limit = false. so check have output distinct rows, not _output_total_rows
-            if (_has_global_limit &&
-                (current_output_rows + _output_total_rows) >= _partition_inner_limit) {
-                get_enough_data = true;
-                break;
-            }
-            if (_has_global_limit) {
-                current_output_rows++;
-            } else {
-                bool cmp_res = _previous_row->compare_two_rows(current);
-                //get a distinct row
-                if (cmp_res == false) {
-                    _output_distinct_rows++; //need rows++ firstly
-                    if (_output_distinct_rows >= _partition_inner_limit) {
-                        get_enough_data = true;
-                        break;
-                    }
-                    *_previous_row = current;
-                }
-            }
+
+        if (step) {
+            merged_rows += step;
+            _output_total_rows += step;
             for (size_t i = 0; i < num_columns; ++i) {
-                merged_columns[i]->insert_from(*current->block->get_columns()[i], current->pos);
+                merged_columns[i]->insert_range_from(*current->impl->columns[i], current->impl->pos,
+                                                     step);
             }
-            break;
         }
-        case TopNAlgorithm::RANK: {
-            //  rank(): 1,1,1,4,5,6,6,6.....,6,100,101. if SQL where rk < 7, need output all 1,1,1,4,5,6,6,....6
-            //2 rank() maybe need check when have get a distinct row
-            //2.1 _has_global_limit = true: (current_output_rows + _output_total_rows) >= _partition_inner_limit)
-            //2.2 _has_global_limit = false: so when the cmp_res is get a distinct row, need check have output all rows num
-            if (_has_global_limit &&
-                (current_output_rows + _output_total_rows) >= _partition_inner_limit) {
-                get_enough_data = true;
-                break;
-            }
-            bool cmp_res = _previous_row->compare_two_rows(current);
-            //get a distinct row
-            if (cmp_res == false) {
-                //here must be check distinct of two rows, and then check nums of row
-                if ((current_output_rows + _output_total_rows) >= _partition_inner_limit) {
-                    get_enough_data = true;
-                    break;
+
+        if (!current->impl->is_last(step)) {
+            queue.next(step);
+        } else {
+            queue.remove_top();
+        }
+    }
+
+    return Status::OK();
+}
+
+Status PartitionSorter::_read_row_rank(Block* output_block, bool* eos, int batch_size) {
+    auto& queue = _state->get_queue();
+    size_t num_columns = _state->unsorted_block()->columns();
+
+    MutableBlock m_block =
+            VectorizedUtils::build_mutable_mem_reuse_block(output_block, *_state->unsorted_block());
+    MutableColumns& merged_columns = m_block.mutable_columns();
+    size_t merged_rows = 0;
+
+    Defer defer {[&]() {
+        if (merged_rows == 0 || _get_enough_data()) {
+            *eos = true;
+        }
+    }};
+
+    while (queue.is_valid() && merged_rows < batch_size) {
+        auto [current, current_rows] = queue.current();
+
+        for (size_t offset = 0; offset < current_rows && merged_rows < batch_size; offset++) {
+            bool cmp_res = _previous_row->impl && _previous_row->compare_two_rows(current->impl);
+            if (!cmp_res) {
+                // 1. dense_rank(): 1,1,1,2,2,2,2,.......,2,3,3,3, if SQL: where rk < 3, need output all 1 and 2
+                // dense_rank() maybe need distinct rows of partition_inner_limit
+                // so check have output distinct rows, not _output_total_rows
+                // 2. rank(): 1,1,1,4,5,6,6,6.....,6,100,101. if SQL where rk < 7, need output all 1,1,1,4,5,6,6,....6
+                // rank() maybe need check when have get a distinct row
+                // so when the cmp_res is get a distinct row, need check have output all rows num
+                if (_get_enough_data()) {
+                    return Status::OK();
                 }
-                *_previous_row = current;
+                *_previous_row = *current;
+                _output_distinct_rows++;
             }
             for (size_t i = 0; i < num_columns; ++i) {
-                merged_columns[i]->insert_from(*current->block->get_columns()[i], current->pos);
+                merged_columns[i]->insert_from(*current->impl->block->get_columns()[i],
+                                               current->impl->pos);
+            }
+            merged_rows++;
+            _output_total_rows++;
+            if (!current->impl->is_last(1)) {
+                queue.next(1);
+            } else {
+                queue.remove_top();
             }
-            current_output_rows++;
-            break;
-        }
-        default:
-            break;
-        }
-
-        if (!current->is_last()) {
-            current->next();
-            priority_queue.push(current);
-        }
-
-        if (current_output_rows == batch_size || get_enough_data == true) {
-            break;
         }
     }
 
-    _output_total_rows += output_block->rows();
-    if (current_output_rows == 0 || get_enough_data == true) {
-        *eos = true;
-    }
     return Status::OK();
 }
 
diff --git a/be/src/vec/common/sort/partition_sorter.h b/be/src/vec/common/sort/partition_sorter.h
index 0939dcd40cdfdb9..053b3aa1a29bbdc 100644
--- a/be/src/vec/common/sort/partition_sorter.h
+++ b/be/src/vec/common/sort/partition_sorter.h
@@ -90,17 +90,30 @@ class PartitionSorter final : public Sorter {
     Status get_next(RuntimeState* state, Block* block, bool* eos) override;
 
     size_t data_size() const override { return _state->data_size(); }
-
-    Status partition_sort_read(Block* block, bool* eos, int batch_size);
     int64 get_output_rows() const { return _output_total_rows; }
     void reset_sorter_state(RuntimeState* runtime_state);
 
 private:
+    Status _read_row_num(Block* block, bool* eos, int batch_size);
+    Status _read_row_rank(Block* block, bool* eos, int batch_size);
+    bool _get_enough_data() const {
+        if (_top_n_algorithm == TopNAlgorithm::DENSE_RANK) {
+            // dense_rank(): 1,1,1,2,2,2,2,.......,2,3,3,3, if SQL: where rk < 3, need output all 1 and 2
+            // dense_rank() maybe need distinct rows of partition_inner_limit
+            // so check have output distinct rows, not _output_total_rows
+            return _output_distinct_rows >= _partition_inner_limit;
+        } else {
+            // rank(): 1,1,1,4,5,6,6,6.....,6,100,101. if SQL where rk < 7, need output all 1,1,1,4,5,6,6,....6
+            // rank() maybe need check when have get a distinct row
+            // so when the cmp_res is get a distinct row, need check have output all rows num
+            return _output_total_rows >= _partition_inner_limit;
+        }
+    }
+
     std::unique_ptr<MergeSorterState> _state;
     const RowDescriptor& _row_desc;
     int64 _output_total_rows = 0;
     int64 _output_distinct_rows = 0;
-    bool _has_global_limit = false;
     int _partition_inner_limit = 0;
     TopNAlgorithm::type _top_n_algorithm = TopNAlgorithm::type::ROW_NUMBER;
     SortCursorCmp* _previous_row = nullptr;
diff --git a/be/src/vec/common/sort/sorter.cpp b/be/src/vec/common/sort/sorter.cpp
index 4f7de1d379aea95..0fddcf011820ef4 100644
--- a/be/src/vec/common/sort/sorter.cpp
+++ b/be/src/vec/common/sort/sorter.cpp
@@ -57,13 +57,11 @@ namespace doris::vectorized {
 //
 
 void MergeSorterState::reset() {
-    auto empty_queue = std::priority_queue<MergeSortCursor>();
-    priority_queue_.swap(empty_queue);
     std::vector<std::shared_ptr<MergeSortCursorImpl>> empty_cursors(0);
     std::vector<std::shared_ptr<Block>> empty_blocks(0);
-    sorted_blocks_.swap(empty_blocks);
-    unsorted_block_ = Block::create_unique(unsorted_block_->clone_empty());
-    in_mem_sorted_bocks_size_ = 0;
+    _sorted_blocks.swap(empty_blocks);
+    unsorted_block() = Block::create_unique(unsorted_block()->clone_empty());
+    _in_mem_sorted_bocks_size = 0;
 }
 
 void MergeSorterState::add_sorted_block(std::shared_ptr<Block> block) {
@@ -71,72 +69,80 @@ void MergeSorterState::add_sorted_block(std::shared_ptr<Block> block) {
     if (0 == rows) {
         return;
     }
-    in_mem_sorted_bocks_size_ += block->bytes();
-    sorted_blocks_.emplace_back(block);
-    num_rows_ += rows;
+    _in_mem_sorted_bocks_size += block->bytes();
+    _sorted_blocks.emplace_back(block);
+    _num_rows += rows;
 }
 
 Status MergeSorterState::build_merge_tree(const SortDescription& sort_description) {
-    for (auto& block : sorted_blocks_) {
-        priority_queue_.emplace(
+    std::vector<MergeSortCursor> cursors;
+    for (auto& block : _sorted_blocks) {
+        cursors.emplace_back(
                 MergeSortCursorImpl::create_shared(std::move(block), sort_description));
     }
+    _queue = MergeSorterQueue(cursors);
 
-    sorted_blocks_.clear();
+    _sorted_blocks.clear();
     return Status::OK();
 }
 
 Status MergeSorterState::merge_sort_read(doris::vectorized::Block* block, int batch_size,
                                          bool* eos) {
-    DCHECK(sorted_blocks_.empty());
-    DCHECK(unsorted_block_->empty());
-    if (priority_queue_.empty()) {
-        *eos = true;
-    } else if (priority_queue_.size() == 1) {
-        if (offset_ != 0 || priority_queue_.top()->pos != 0) {
-            // Skip rows already returned or need to be ignored
-            int64_t offset = offset_ + (int64_t)priority_queue_.top()->pos;
-            priority_queue_.top().impl->block->skip_num_rows(offset);
-        }
-        block->swap(*priority_queue_.top().impl->block);
-        *eos = true;
-    } else {
-        RETURN_IF_ERROR(_merge_sort_read_impl(batch_size, block, eos));
-    }
+    DCHECK(_sorted_blocks.empty());
+    DCHECK(unsorted_block()->empty());
+    RETURN_IF_ERROR(_merge_sort_read_impl(batch_size, block, eos));
     return Status::OK();
 }
 
 Status MergeSorterState::_merge_sort_read_impl(int batch_size, doris::vectorized::Block* block,
                                                bool* eos) {
-    size_t num_columns = priority_queue_.top().impl->block->columns();
+    size_t num_columns = unsorted_block()->columns();
 
-    MutableBlock m_block = VectorizedUtils::build_mutable_mem_reuse_block(
-            block, *priority_queue_.top().impl->block);
+    MutableBlock m_block = VectorizedUtils::build_mutable_mem_reuse_block(block, *unsorted_block());
     MutableColumns& merged_columns = m_block.mutable_columns();
 
     /// Take rows from queue in right order and push to 'merged'.
     size_t merged_rows = 0;
     // process single element queue on merge_sort_read()
-    while (priority_queue_.size() > 1 && merged_rows < batch_size) {
-        auto current = priority_queue_.top();
-        priority_queue_.pop();
+    while (_queue.is_valid() && merged_rows < batch_size) {
+        auto [current, current_rows] = _queue.current();
+        current_rows = std::min(current_rows, batch_size - merged_rows);
+
+        size_t step = std::min(_offset, current_rows);
+        _offset -= step;
+        current_rows -= step;
+
+        if (current->impl->is_last(current_rows + step) && current->impl->pos == 0 && step == 0) {
+            if (merged_rows != 0) {
+                // return directly for next time's read swap whole block
+                return Status::OK();
+            }
+            // swap and return block directly when we should get all data from cursor
+            block->swap(*current->impl->block);
+            _queue.remove_top();
+            return Status::OK();
+        }
 
-        if (offset_ == 0) {
+        if (current_rows) {
             for (size_t i = 0; i < num_columns; ++i) {
-                merged_columns[i]->insert_from(*current->block->get_columns()[i], current->pos);
+                merged_columns[i]->insert_range_from(*current->impl->columns[i],
+                                                     current->impl->pos + step, current_rows);
             }
-            ++merged_rows;
-        } else {
-            offset_--;
+            merged_rows += current_rows;
         }
 
-        if (!current->is_last()) {
-            current->next();
-            priority_queue_.push(current);
+        if (!current->impl->is_last(current_rows + step)) {
+            _queue.next(current_rows + step);
+        } else {
+            _queue.remove_top();
         }
     }
 
     block->set_columns(std::move(merged_columns));
+
+    if (merged_rows == 0) {
+        *eos = true;
+    }
     return Status::OK();
 }
 
@@ -207,23 +213,28 @@ FullSorter::FullSorter(VSortExecExprs& vsort_exec_exprs, int limit, int64_t offs
 Status FullSorter::append_block(Block* block) {
     DCHECK(block->rows() > 0);
 
-    if (_reach_limit() && block->bytes() > _state->unsorted_block_->allocated_bytes() -
-                                                   _state->unsorted_block_->bytes()) {
+    if (_reach_limit() && block->bytes() > _state->unsorted_block()->allocated_bytes() -
+                                                   _state->unsorted_block()->bytes()) {
         RETURN_IF_ERROR(_do_sort());
     }
 
     {
         SCOPED_TIMER(_merge_block_timer);
-        const auto& data = _state->unsorted_block_->get_columns_with_type_and_name();
+        const auto& data = _state->unsorted_block()->get_columns_with_type_and_name();
         const auto& arrival_data = block->get_columns_with_type_and_name();
         auto sz = block->rows();
         for (int i = 0; i < data.size(); ++i) {
             DCHECK(data[i].type->equals(*(arrival_data[i].type)))
                     << " type1: " << data[i].type->get_name()
                     << " type2: " << arrival_data[i].type->get_name() << " i: " << i;
-            //TODO: to eliminate unnecessary expansion, we need a `insert_range_from_const` for every column type.
-            data[i].column->assume_mutable()->insert_range_from(
-                    *arrival_data[i].column->convert_to_full_column_if_const(), 0, sz);
+            if (is_column_const(*arrival_data[i].column)) {
+                data[i].column->assume_mutable()->insert_many_from(
+                        assert_cast<const ColumnConst*>(arrival_data[i].column.get())
+                                ->get_data_column(),
+                        0, sz);
+            } else {
+                data[i].column->assume_mutable()->insert_range_from(*arrival_data[i].column, 0, sz);
+            }
         }
         block->clear_column_data();
     }
@@ -231,7 +242,7 @@ Status FullSorter::append_block(Block* block) {
 }
 
 Status FullSorter::prepare_for_read() {
-    if (_state->unsorted_block_->rows() > 0) {
+    if (_state->unsorted_block()->rows() > 0) {
         RETURN_IF_ERROR(_do_sort());
     }
     return _state->build_merge_tree(_sort_description);
@@ -247,7 +258,7 @@ Status FullSorter::merge_sort_read_for_spill(RuntimeState* state, doris::vectori
 }
 
 Status FullSorter::_do_sort() {
-    Block* src_block = _state->unsorted_block_.get();
+    Block* src_block = _state->unsorted_block().get();
     Block desc_block = src_block->clone_without_columns();
     RETURN_IF_ERROR(partial_sort(*src_block, desc_block));
 
diff --git a/be/src/vec/common/sort/sorter.h b/be/src/vec/common/sort/sorter.h
index 36c535c9101db93..69e70ab869cd4be 100644
--- a/be/src/vec/common/sort/sorter.h
+++ b/be/src/vec/common/sort/sorter.h
@@ -43,6 +43,8 @@ class RowDescriptor;
 
 namespace doris::vectorized {
 
+using MergeSorterQueue = SortingQueueBatch<MergeSortCursor>;
+
 // TODO: now we only use merge sort
 class MergeSorterState {
     ENABLE_FACTORY_CREATOR(MergeSorterState);
@@ -53,9 +55,9 @@ class MergeSorterState {
             // create_empty_block should ignore invalid slots, unsorted_block
             // should be same structure with arrival block from child node
             // since block from child node may ignored these slots
-            : unsorted_block_(Block::create_unique(
+            : _unsorted_block(Block::create_unique(
                       VectorizedUtils::create_empty_block(row_desc, true /*ignore invalid slot*/))),
-              offset_(offset) {}
+              _offset(offset) {}
 
     ~MergeSorterState() = default;
 
@@ -66,32 +68,33 @@ class MergeSorterState {
     Status merge_sort_read(doris::vectorized::Block* block, int batch_size, bool* eos);
 
     size_t data_size() const {
-        size_t size = unsorted_block_->bytes();
-        return size + in_mem_sorted_bocks_size_;
+        size_t size = _unsorted_block->bytes();
+        return size + _in_mem_sorted_bocks_size;
     }
 
-    uint64_t num_rows() const { return num_rows_; }
+    uint64_t num_rows() const { return _num_rows; }
 
-    std::shared_ptr<Block> last_sorted_block() { return sorted_blocks_.back(); }
+    std::shared_ptr<Block> last_sorted_block() { return _sorted_blocks.back(); }
 
-    std::vector<std::shared_ptr<Block>>& get_sorted_block() { return sorted_blocks_; }
-    std::priority_queue<MergeSortCursor>& get_priority_queue() { return priority_queue_; }
+    std::vector<std::shared_ptr<Block>>& get_sorted_block() { return _sorted_blocks; }
+    MergeSorterQueue& get_queue() { return _queue; }
     void reset();
 
-    std::unique_ptr<Block> unsorted_block_;
+    std::unique_ptr<Block>& unsorted_block() { return _unsorted_block; }
 
 private:
     Status _merge_sort_read_impl(int batch_size, doris::vectorized::Block* block, bool* eos);
 
-    std::priority_queue<MergeSortCursor> priority_queue_;
-    std::vector<std::shared_ptr<Block>> sorted_blocks_;
-    size_t in_mem_sorted_bocks_size_ = 0;
-    uint64_t num_rows_ = 0;
+    std::unique_ptr<Block> _unsorted_block;
+    MergeSorterQueue _queue;
+    std::vector<std::shared_ptr<Block>> _sorted_blocks;
+    size_t _in_mem_sorted_bocks_size = 0;
+    uint64_t _num_rows = 0;
 
-    int64_t offset_;
+    size_t _offset;
 
-    Block merge_sorted_block_;
-    std::unique_ptr<VSortedRunMerger> merger_;
+    Block _merge_sorted_block;
+    std::unique_ptr<VSortedRunMerger> _merger;
 };
 
 class Sorter {
@@ -177,7 +180,7 @@ class FullSorter final : public Sorter {
 
 private:
     bool _reach_limit() {
-        return _state->unsorted_block_->allocated_bytes() >= buffered_block_bytes_;
+        return _state->unsorted_block()->allocated_bytes() >= buffered_block_bytes_;
     }
 
     Status _do_sort();
diff --git a/be/src/vec/core/sort_cursor.h b/be/src/vec/core/sort_cursor.h
index d31767f46e461ff..a37b6feb21ecd66 100644
--- a/be/src/vec/core/sort_cursor.h
+++ b/be/src/vec/core/sort_cursor.h
@@ -20,6 +20,8 @@
 
 #pragma once
 
+#include <utility>
+
 #include "vec/columns/column.h"
 #include "vec/core/block.h"
 #include "vec/core/sort_description.h"
@@ -48,8 +50,7 @@ struct HeapSortCursorBlockView {
     void _reset() {
         sort_columns.clear();
         auto columns = block.get_columns_and_convert();
-        for (size_t j = 0, size = desc.size(); j < size; ++j) {
-            auto& column_desc = desc[j];
+        for (auto& column_desc : desc) {
             size_t column_number = !column_desc.column_name.empty()
                                            ? block.get_position_by_name(column_desc.column_name)
                                            : column_desc.column_number;
@@ -63,7 +64,7 @@ using HeapSortCursorBlockSPtr = std::shared_ptr<HeapSortCursorBlockView>;
 struct HeapSortCursorImpl {
 public:
     HeapSortCursorImpl(int row_id, HeapSortCursorBlockSPtr block_view)
-            : _row_id(row_id), _block_view(block_view) {}
+            : _row_id(row_id), _block_view(std::move(block_view)) {}
 
     HeapSortCursorImpl(const HeapSortCursorImpl& other) {
         _row_id = other._row_id;
@@ -123,6 +124,7 @@ struct MergeSortCursorImpl {
     ENABLE_FACTORY_CREATOR(MergeSortCursorImpl);
     std::shared_ptr<Block> block;
     ColumnRawPtrs sort_columns;
+    ColumnRawPtrs columns;
     SortDescription desc;
     size_t sort_columns_size = 0;
     size_t pos = 0;
@@ -131,26 +133,33 @@ struct MergeSortCursorImpl {
     MergeSortCursorImpl() = default;
     virtual ~MergeSortCursorImpl() = default;
 
-    MergeSortCursorImpl(std::shared_ptr<Block> block_, const SortDescription& desc_)
-            : block(block_), desc(desc_), sort_columns_size(desc.size()) {
+    MergeSortCursorImpl(std::shared_ptr<Block> block_, SortDescription desc_)
+            : block(std::move(block_)), desc(std::move(desc_)), sort_columns_size(desc.size()) {
         reset();
     }
 
-    MergeSortCursorImpl(const SortDescription& desc_)
-            : block(Block::create_shared()), desc(desc_), sort_columns_size(desc.size()) {}
+    MergeSortCursorImpl(SortDescription desc_)
+            : block(Block::create_shared()),
+              desc(std::move(desc_)),
+              sort_columns_size(desc.size()) {}
+
     bool empty() const { return rows == 0; }
 
     /// Set the cursor to the beginning of the new block.
     void reset() {
         sort_columns.clear();
+        columns.clear();
 
-        auto columns = block->get_columns_and_convert();
-        for (size_t j = 0, size = desc.size(); j < size; ++j) {
-            auto& column_desc = desc[j];
+        auto tmp_columns = block->get_columns_and_convert();
+        columns.reserve(tmp_columns.size());
+        for (auto col : tmp_columns) {
+            columns.push_back(col.get());
+        }
+        for (auto& column_desc : desc) {
             size_t column_number = !column_desc.column_name.empty()
                                            ? block->get_position_by_name(column_desc.column_name)
                                            : column_desc.column_number;
-            sort_columns.push_back(columns[column_number].get());
+            sort_columns.push_back(columns[column_number]);
         }
 
         pos = 0;
@@ -158,8 +167,9 @@ struct MergeSortCursorImpl {
     }
 
     bool is_first() const { return pos == 0; }
-    bool is_last() const { return pos + 1 >= rows; }
-    void next() { ++pos; }
+    bool is_last(size_t size = 1) const { return pos + size >= rows; }
+    void next(size_t size = 1) { pos += size; }
+    size_t get_size() const { return rows; }
 
     virtual bool has_next_block() { return false; }
     virtual Block* block_ptr() { return nullptr; }
@@ -169,11 +179,11 @@ using BlockSupplier = std::function<Status(Block*, bool* eos)>;
 
 struct BlockSupplierSortCursorImpl : public MergeSortCursorImpl {
     ENABLE_FACTORY_CREATOR(BlockSupplierSortCursorImpl);
-    BlockSupplierSortCursorImpl(const BlockSupplier& block_supplier,
+    BlockSupplierSortCursorImpl(BlockSupplier block_supplier,
                                 const VExprContextSPtrs& ordering_expr,
                                 const std::vector<bool>& is_asc_order,
                                 const std::vector<bool>& nulls_first)
-            : _ordering_expr(ordering_expr), _block_supplier(block_supplier) {
+            : _ordering_expr(ordering_expr), _block_supplier(std::move(block_supplier)) {
         block = Block::create_shared();
         sort_columns_size = ordering_expr.size();
 
@@ -185,8 +195,8 @@ struct BlockSupplierSortCursorImpl : public MergeSortCursorImpl {
         _is_eof = !has_next_block();
     }
 
-    BlockSupplierSortCursorImpl(const BlockSupplier& block_supplier, const SortDescription& desc_)
-            : MergeSortCursorImpl(desc_), _block_supplier(block_supplier) {
+    BlockSupplierSortCursorImpl(BlockSupplier block_supplier, const SortDescription& desc_)
+            : MergeSortCursorImpl(desc_), _block_supplier(std::move(block_supplier)) {
         _is_eof = !has_next_block();
     }
 
@@ -202,7 +212,7 @@ struct BlockSupplierSortCursorImpl : public MergeSortCursorImpl {
         // If status not ok, upper callers could not detect whether it is eof or error.
         // So that fatal here, and should throw exception in the future.
         if (status.ok() && !block->empty()) {
-            if (_ordering_expr.size() > 0) {
+            if (!_ordering_expr.empty()) {
                 for (int i = 0; status.ok() && i < desc.size(); ++i) {
                     // TODO yiguolei: throw exception if status not ok in the future
                     status = _ordering_expr[i]->execute(block.get(), &desc[i].column_number);
@@ -233,7 +243,7 @@ struct MergeSortCursor {
     ENABLE_FACTORY_CREATOR(MergeSortCursor);
     std::shared_ptr<MergeSortCursorImpl> impl;
 
-    MergeSortCursor(std::shared_ptr<MergeSortCursorImpl> impl_) : impl(impl_) {}
+    MergeSortCursor(std::shared_ptr<MergeSortCursorImpl> impl_) : impl(std::move(impl_)) {}
     MergeSortCursorImpl* operator->() const { return impl.get(); }
 
     /// The specified row of this cursor is greater than the specified row of another cursor.
@@ -264,6 +274,21 @@ struct MergeSortCursor {
         return greater_at(rhs, impl->rows - 1, 0) == -1;
     }
 
+    /// Checks that all rows in the current block of this cursor are less than or equal to all the rows of the current block of another cursor.
+    bool totally_less_or_equals(const MergeSortCursor& rhs) const {
+        if (impl->rows == 0 || rhs.impl->rows == 0) {
+            return false;
+        }
+
+        /// The last row of this cursor is no larger than the first row of the another cursor.
+        return greater_at(rhs, impl->rows - 1, rhs->pos) <= 0;
+    }
+
+    bool greater_with_offset(const MergeSortCursor& rhs, size_t lhs_offset,
+                             size_t rhs_offset) const {
+        return greater_at(rhs, impl->pos + lhs_offset, rhs.impl->pos + rhs_offset) > 0;
+    }
+
     bool greater(const MergeSortCursor& rhs) const {
         return !impl->empty() && greater_at(rhs, impl->pos, rhs.impl->pos) > 0;
     }
@@ -277,7 +302,7 @@ struct MergeSortBlockCursor {
     ENABLE_FACTORY_CREATOR(MergeSortBlockCursor);
     std::shared_ptr<MergeSortCursorImpl> impl = nullptr;
 
-    MergeSortBlockCursor(std::shared_ptr<MergeSortCursorImpl> impl_) : impl(impl_) {}
+    MergeSortBlockCursor(std::shared_ptr<MergeSortCursorImpl> impl_) : impl(std::move(impl_)) {}
     MergeSortCursorImpl* operator->() const { return impl.get(); }
 
     /// The specified row of this cursor is greater than the specified row of another cursor.
@@ -314,4 +339,219 @@ struct MergeSortBlockCursor {
     }
 };
 
+enum class SortingQueueStrategy : uint8_t { Default, Batch };
+
+/// Allows to fetch data from multiple sort cursors in sorted order (merging sorted data streams).
+template <typename Cursor, SortingQueueStrategy strategy>
+class SortingQueueImpl {
+public:
+    SortingQueueImpl() = default;
+
+    template <typename Cursors>
+    explicit SortingQueueImpl(Cursors& cursors) {
+        size_t size = cursors.size();
+        _queue.reserve(size);
+
+        for (size_t i = 0; i < size; ++i) {
+            _queue.emplace_back(cursors[i]);
+        }
+
+        std::make_heap(_queue.begin(), _queue.end());
+
+        if constexpr (strategy == SortingQueueStrategy::Batch) {
+            if (!_queue.empty()) {
+                update_batch_size();
+            }
+        }
+    }
+
+    bool is_valid() const { return !_queue.empty(); }
+
+    Cursor& current()
+        requires(strategy == SortingQueueStrategy::Default)
+    {
+        return &_queue.front();
+    }
+
+    std::pair<Cursor*, size_t> current()
+        requires(strategy == SortingQueueStrategy::Batch)
+    {
+        return {&_queue.front(), batch_size};
+    }
+
+    size_t size() { return _queue.size(); }
+
+    Cursor& next_child() { return _queue[next_child_index()]; }
+
+    void ALWAYS_INLINE next()
+        requires(strategy == SortingQueueStrategy::Default)
+    {
+        assert(is_valid());
+
+        if (!_queue.front()->is_last()) {
+            _queue.front()->next();
+            update_top(true);
+        } else {
+            remove_top();
+        }
+    }
+
+    void ALWAYS_INLINE next(size_t batch_size_value)
+        requires(strategy == SortingQueueStrategy::Batch)
+    {
+        assert(is_valid());
+        assert(batch_size_value <= batch_size);
+        assert(batch_size_value > 0);
+
+        batch_size -= batch_size_value;
+        if (batch_size > 0) {
+            _queue.front()->next(batch_size_value);
+            return;
+        }
+
+        if (!_queue.front()->is_last(batch_size_value)) {
+            _queue.front()->next(batch_size_value);
+            update_top(false);
+        } else {
+            remove_top();
+        }
+    }
+
+    void remove_top() {
+        std::pop_heap(_queue.begin(), _queue.end());
+        _queue.pop_back();
+        next_child_idx = 0;
+
+        if constexpr (strategy == SortingQueueStrategy::Batch) {
+            if (_queue.empty()) {
+                batch_size = 0;
+            } else {
+                update_batch_size();
+            }
+        }
+    }
+
+    void push(MergeSortCursorImpl& cursor) {
+        _queue.emplace_back(&cursor);
+        std::push_heap(_queue.begin(), _queue.end());
+        next_child_idx = 0;
+
+        if constexpr (strategy == SortingQueueStrategy::Batch) {
+            update_batch_size();
+        }
+    }
+
+private:
+    using Container = std::vector<Cursor>;
+    Container _queue;
+
+    /// Cache comparison between first and second child if the order in queue has not been changed.
+    size_t next_child_idx = 0;
+    size_t batch_size = 0;
+
+    size_t ALWAYS_INLINE next_child_index() {
+        if (next_child_idx == 0) {
+            next_child_idx = 1;
+
+            if (_queue.size() > 2 && _queue[1].greater(_queue[2])) {
+                ++next_child_idx;
+            }
+        }
+
+        return next_child_idx;
+    }
+
+    /// This is adapted version of the function __sift_down from libc++.
+    /// Why cannot simply use std::priority_queue?
+    /// - because it doesn't support updating the top element and requires pop and push instead.
+    /// Also look at "Boost.Heap" library.
+    void ALWAYS_INLINE update_top(bool check_in_order) {
+        size_t size = _queue.size();
+        if (size < 2) {
+            return;
+        }
+
+        auto begin = _queue.begin();
+
+        size_t child_idx = next_child_index();
+        auto child_it = begin + child_idx;
+
+        /// Check if we are in order.
+        if (check_in_order && (*child_it).greater(*begin)) {
+            if constexpr (strategy == SortingQueueStrategy::Batch) {
+                update_batch_size();
+            }
+            return;
+        }
+
+        next_child_idx = 0;
+
+        auto curr_it = begin;
+        auto top(std::move(*begin));
+        do {
+            /// We are not in heap-order, swap the parent with it's largest child.
+            *curr_it = std::move(*child_it);
+            curr_it = child_it;
+
+            // recompute the child based off of the updated parent
+            child_idx = 2 * child_idx + 1;
+
+            if (child_idx >= size) {
+                break;
+            }
+
+            child_it = begin + child_idx;
+
+            if ((child_idx + 1) < size && (*child_it).greater(*(child_it + 1))) {
+                /// Right child exists and is greater than left child.
+                ++child_it;
+                ++child_idx;
+            }
+
+            /// Check if we are in order.
+        } while (!((*child_it).greater(top)));
+        *curr_it = std::move(top);
+
+        if constexpr (strategy == SortingQueueStrategy::Batch) {
+            update_batch_size();
+        }
+    }
+
+    /// Update batch size of elements that client can extract from current cursor
+    void update_batch_size() {
+        DCHECK(!_queue.empty());
+
+        auto& begin_cursor = *_queue.begin();
+        size_t min_cursor_size = begin_cursor->get_size();
+        size_t min_cursor_pos = begin_cursor->pos;
+
+        if (_queue.size() == 1) {
+            batch_size = min_cursor_size - min_cursor_pos;
+            return;
+        }
+
+        batch_size = 1;
+        size_t child_idx = next_child_index();
+        auto& next_child_cursor = *(_queue.begin() + child_idx);
+        if (min_cursor_pos + batch_size < min_cursor_size &&
+            next_child_cursor.greater_with_offset(begin_cursor, 0, batch_size)) {
+            ++batch_size;
+        } else {
+            return;
+        }
+        if (begin_cursor.totally_less_or_equals(next_child_cursor)) {
+            batch_size = min_cursor_size - min_cursor_pos;
+            return;
+        }
+
+        while (min_cursor_pos + batch_size < min_cursor_size &&
+               next_child_cursor.greater_with_offset(begin_cursor, 0, batch_size)) {
+            ++batch_size;
+        }
+    }
+};
+template <typename Cursor>
+using SortingQueue = SortingQueueImpl<Cursor, SortingQueueStrategy::Default>;
+template <typename Cursor>
+using SortingQueueBatch = SortingQueueImpl<Cursor, SortingQueueStrategy::Batch>;
 } // namespace doris::vectorized
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PartitionSortNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PartitionSortNode.java
index 20142e380ce7105..69a1b871d1ba665 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/PartitionSortNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PartitionSortNode.java
@@ -142,7 +142,10 @@ protected void toThrift(TPlanNode msg) {
         Preconditions.checkState(tupleIds.size() == 1, "Incorrect size for tupleIds in PartitionSortNode");
 
         TopNAlgorithm topNAlgorithm;
-        if (function == WindowFuncType.ROW_NUMBER) {
+        if (hasGlobalLimit) {
+            // only need row number if has global limit, so we change algorithm directly
+            topNAlgorithm = TopNAlgorithm.ROW_NUMBER;
+        } else if (function == WindowFuncType.ROW_NUMBER) {
             topNAlgorithm = TopNAlgorithm.ROW_NUMBER;
         } else if (function == WindowFuncType.RANK) {
             topNAlgorithm = TopNAlgorithm.RANK;

From c9f19a677acd8532b30e2b89c7f5db0a4c726c2e Mon Sep 17 00:00:00 2001
From: bobhan1 <baohan@selectdb.com>
Date: Tue, 7 Jan 2025 15:19:17 +0800
Subject: [PATCH 020/140] [opt](flexible partial update) Don't store table
 property `enable_unique_key_skip_bitmap_column` (#46273)

Problem Summary:

When creating table, table property
`enable_unique_key_skip_bitmap_column` indicate whether or not to add
the hidden skip bitmap column for the targeted table. After the table is
created, this property indicate that whether the table has the hidden
skip bitmap column. So we don't need to actually store this property in
table's property map and don't need to pay for keeping consistency
between these two things.
---
 .../doris/alter/SchemaChangeHandler.java      | 54 +------------------
 .../apache/doris/alter/SchemaChangeJobV2.java | 14 -----
 .../apache/doris/analysis/AlterTableStmt.java |  4 +-
 .../doris/analysis/CreateTableStmt.java       |  2 +-
 .../analysis/ModifyTablePropertiesClause.java |  2 +-
 .../catalog/InternalSchemaInitializer.java    |  1 -
 .../org/apache/doris/catalog/OlapTable.java   |  9 +---
 .../apache/doris/catalog/TableProperty.java   |  4 --
 .../doris/datasource/InternalCatalog.java     |  5 +-
 .../plans/commands/info/CreateTableInfo.java  |  2 +-
 .../info/ModifyTablePropertiesOp.java         |  2 +-
 .../query_p0/system/test_table_properties.out |  8 +--
 ...st_flexible_partial_update_property.groovy |  5 ++
 13 files changed, 16 insertions(+), 96 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index a97c15bfc0c02ef..72d2419177c1892 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -1385,26 +1385,6 @@ private void createJob(String rawSql, long dbId, OlapTable olapTable, Map<Long,
             }
         }
 
-        // property enable_unique_key_skip_bitmap
-        boolean enableUniqueKeySkipBitmap = false;
-        boolean hasEnableUniqueKeySkipBitmapChanged = false;
-        if (propertyMap.containsKey(PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN)) {
-            if (!(olapTable.getKeysType() == KeysType.UNIQUE_KEYS && olapTable.getEnableUniqueKeyMergeOnWrite())) {
-                throw new DdlException("Only allow to alter table property enable_unique_key_skip_bitmap"
-                        + "on unique merge-on-write table.");
-            }
-            try {
-                enableUniqueKeySkipBitmap = PropertyAnalyzer.analyzeUniqueKeySkipBitmapColumn(propertyMap);
-            } catch (AnalysisException e) {
-                throw new DdlException(e.getMessage());
-            }
-            if (enableUniqueKeySkipBitmap) {
-                if (olapTable.getEnableUniqueKeySkipBitmap() != enableUniqueKeySkipBitmap) {
-                    hasEnableUniqueKeySkipBitmapChanged = true;
-                }
-            }
-        }
-
         // begin checking each table
         // ATTN: DO NOT change any meta in this loop
         long tableId = olapTable.getId();
@@ -1480,8 +1460,6 @@ private void createJob(String rawSql, long dbId, OlapTable olapTable, Map<Long,
                 if (olapTable.getStorageFormat() != TStorageFormat.V2) {
                     needAlter = true;
                 }
-            } else if (hasEnableUniqueKeySkipBitmapChanged) {
-                needAlter = true;
             }
 
             if (!needAlter) {
@@ -1599,8 +1577,7 @@ private void createJob(String rawSql, long dbId, OlapTable olapTable, Map<Long,
             }
         } // end for indices
 
-        if (changedIndexIdToSchema.isEmpty() && !hasIndexChange && !hasRowStoreChanged
-                && !hasEnableUniqueKeySkipBitmapChanged) {
+        if (changedIndexIdToSchema.isEmpty() && !hasIndexChange && !hasRowStoreChanged) {
             throw new DdlException("Nothing is changed. please check your alter stmt.");
         }
 
@@ -1614,8 +1591,6 @@ private void createJob(String rawSql, long dbId, OlapTable olapTable, Map<Long,
         schemaChangeJob.setBloomFilterInfo(hasBfChange, bfColumns, bfFpp);
         schemaChangeJob.setAlterIndexInfo(hasIndexChange, indexes);
         schemaChangeJob.setStoreRowColumnInfo(hasRowStoreChanged, storeRowColumn, rsColumns);
-        schemaChangeJob.setEnableUniqueKeySkipBitmapInfo(enableUniqueKeySkipBitmap,
-                hasEnableUniqueKeySkipBitmapChanged);
 
         // If StorageFormat is set to TStorageFormat.V2
         // which will create tablet with preferred_rowset_type set to BETA
@@ -2039,22 +2014,6 @@ public int getAsInt() {
                             rowColumn.setUniqueId(maxColUniqueId + 1);
                             indexSchemaMap.get(olapTable.getBaseIndexId()).add(rowColumn);
                         }
-                    } else if (properties.containsKey(PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN)) {
-                        boolean value = Boolean.parseBoolean(properties
-                                .get(PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN));
-                        if (!value) {
-                            throw new DdlException("Can not alter enable_unique_key_skip_bitmap_column"
-                                    + " from true to false currently.");
-                        }
-                        if (!olapTable.hasSkipBitmapColumn()) {
-                            Column skipBitmapColumn =
-                                    ColumnDefinition.newSkipBitmapColumnDef(AggregateType.NONE)
-                                            .translateToCatalogStyle();
-                            int maxColUniqueId = olapTable.getIndexMetaByIndexId(olapTable.getBaseIndexId())
-                                    .getMaxColUniqueId();
-                            skipBitmapColumn.setUniqueId(maxColUniqueId + 1);
-                            indexSchemaMap.get(olapTable.getBaseIndexId()).add(skipBitmapColumn);
-                        }
                     }
                 }
 
@@ -3121,17 +3080,9 @@ public void updateBaseIndexSchema(OlapTable olapTable, Map<Long, LinkedList<Colu
         List<Long> indexIds = new ArrayList<Long>();
         indexIds.add(baseIndexId);
         indexIds.addAll(olapTable.getIndexIdListExceptBaseIndex());
-        boolean hasEnableUniqueKeySkipBitmapChanged = false;
         for (int i = 0; i < indexIds.size(); i++) {
             List<Column> indexSchema = indexSchemaMap.get(indexIds.get(i));
             MaterializedIndexMeta currentIndexMeta = olapTable.getIndexMetaByIndexId(indexIds.get(i));
-            if (i == 0 && !olapTable.getEnableUniqueKeySkipBitmap()) {
-                Preconditions.checkState(!currentIndexMeta.getSchema(true).stream()
-                        .anyMatch(col -> col.isSkipBitmapColumn()));
-                if (indexSchema.stream().anyMatch(col -> col.isSkipBitmapColumn())) {
-                    hasEnableUniqueKeySkipBitmapChanged = true;
-                }
-            }
             currentIndexMeta.setSchema(indexSchema);
 
             int currentSchemaVersion = currentIndexMeta.getSchemaVersion();
@@ -3148,9 +3099,6 @@ public void updateBaseIndexSchema(OlapTable olapTable, Map<Long, LinkedList<Colu
             currentIndexMeta.setMaxColUniqueId(maxColUniqueId);
             currentIndexMeta.setIndexes(indexes);
         }
-        if (hasEnableUniqueKeySkipBitmapChanged) {
-            olapTable.setEnableUniqueKeySkipBitmap(true);
-        }
         olapTable.setIndexes(indexes);
         olapTable.rebuildFullSchema();
         olapTable.rebuildDistributionInfo();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
index 54bcc35262760cf..88fa3286c886bfa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
@@ -143,11 +143,6 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
     @SerializedName(value = "hasRowStoreChange")
     protected boolean hasRowStoreChange = false;
 
-    @SerializedName(value = "enableUniqueKeySkipBitmap")
-    private boolean enableUniqueKeySkipBitmap = false;
-    @SerializedName(value = "hasEnableUniqueKeySkipBitmapChanged")
-    private boolean hasEnableUniqueKeySkipBitmapChanged = false;
-
     // save all schema change tasks
     private AgentBatchTask schemaChangeBatchTask = new AgentBatchTask();
 
@@ -201,12 +196,6 @@ public void setStoreRowColumnInfo(boolean hasRowStoreChange,
         this.rowStoreColumns = rowStoreColumns;
     }
 
-    public void setEnableUniqueKeySkipBitmapInfo(boolean enableUniqueKeySkipBitmap,
-            boolean hasEnableUniqueKeySkipBitmapChanged) {
-        this.enableUniqueKeySkipBitmap = enableUniqueKeySkipBitmap;
-        this.hasEnableUniqueKeySkipBitmapChanged = hasEnableUniqueKeySkipBitmapChanged;
-    }
-
     public void setAlterIndexInfo(boolean indexChange, List<Index> indexes) {
         this.indexChange = indexChange;
         this.indexes = indexes;
@@ -770,9 +759,6 @@ private void onFinished(OlapTable tbl) {
             tbl.setStoreRowColumn(storeRowColumn);
             tbl.setRowStoreColumns(rowStoreColumns);
         }
-        if (hasEnableUniqueKeySkipBitmapChanged) {
-            tbl.setEnableUniqueKeySkipBitmap(enableUniqueKeySkipBitmap);
-        }
 
         // set storage format of table, only set if format is v2
         if (storageFormat == TStorageFormat.V2) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterTableStmt.java
index 9290e0ee89417b0..7738ab0f9f5373b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterTableStmt.java
@@ -158,10 +158,10 @@ public void rewriteAlterClause(OlapTable table) throws UserException {
                         addColumnClause = new AddColumnClause(ColumnDef.newSequenceColumnDef(sequenceColType), null,
                                 null, null);
                     } else if (alterFeature == EnableFeatureClause.Features.UPDATE_FLEXIBLE_COLUMNS) {
-                        ColumnDef skipBItmapCol = ColumnDef.newSkipBitmapColumnDef(AggregateType.NONE);
+                        ColumnDef skipBitmapCol = ColumnDef.newSkipBitmapColumnDef(AggregateType.NONE);
                         List<Column> fullSchema = table.getBaseSchema(true);
                         String lastCol = fullSchema.get(fullSchema.size() - 1).getName();
-                        addColumnClause = new AddColumnClause(skipBItmapCol, new ColumnPosition(lastCol), null, null);
+                        addColumnClause = new AddColumnClause(skipBitmapCol, new ColumnPosition(lastCol), null, null);
                     }
                     addColumnClause.analyze(analyzer);
                     clauses.add(addColumnClause);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
index b9aed862f9bb3a3..5f173b4a243702a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
@@ -429,7 +429,7 @@ public void analyze(Analyzer analyzer) throws UserException {
             if (properties != null) {
                 if (properties.containsKey(PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN)
                         && !(keysDesc.getKeysType() == KeysType.UNIQUE_KEYS && enableUniqueKeyMergeOnWrite)) {
-                    throw new AnalysisException("tablet property enable_unique_key_skip_bitmap_column can"
+                    throw new AnalysisException("table property enable_unique_key_skip_bitmap_column can"
                             + "only be set in merge-on-write unique table.");
                 }
                 // the merge-on-write table must have enable_unique_key_skip_bitmap_column table property
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
index e8d2baa5437c3ec..f2f9a2ac9e47927 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
@@ -375,7 +375,7 @@ public void analyze(Analyzer analyzer) throws AnalysisException {
             this.needTableStable = false;
             this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
         } else if (properties.containsKey(PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN)) {
-            // do nothing, will be analyzed when creating alter job
+            throw new AnalysisException("You can not modify property 'enable_unique_key_skip_bitmap_column'.");
         } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE)) {
             throw new AnalysisException("You can not modify storage_page_size");
         } else {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java
index cfc9d85c6a88c23..4d71ca0e07690c6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java
@@ -276,7 +276,6 @@ private static CreateTableStmt buildStatisticsTblStmt(String statsTableName, Lis
             {
                 put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM, String.valueOf(
                         Math.max(1, Config.min_replication_num_per_tablet)));
-                put(PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN, "false");
             }
         };
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index d30e9f9fde7f24f..9eba9499d637073 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -2856,15 +2856,8 @@ public void setEnableUniqueKeyMergeOnWrite(boolean speedup) {
         getOrCreatTableProperty().setEnableUniqueKeyMergeOnWrite(speedup);
     }
 
-    public void setEnableUniqueKeySkipBitmap(boolean enable) {
-        getOrCreatTableProperty().setEnableUniqueKeySkipBitmap(enable);
-    }
-
     public boolean getEnableUniqueKeySkipBitmap() {
-        if (tableProperty == null) {
-            return false;
-        }
-        return tableProperty.getEnableUniqueKeySkipBitmap();
+        return hasSkipBitmapColumn();
     }
 
     public boolean getEnableUniqueKeyMergeOnWrite() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
index 83ad3b65df2a71a..de839af03cacf8b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
@@ -631,10 +631,6 @@ public void setEnableUniqueKeyMergeOnWrite(boolean enable) {
         properties.put(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE, Boolean.toString(enable));
     }
 
-    public void setEnableUniqueKeySkipBitmap(boolean enable) {
-        properties.put(PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN, Boolean.toString(enable));
-    }
-
     public boolean getEnableUniqueKeySkipBitmap() {
         return Boolean.parseBoolean(properties.getOrDefault(
             PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN, "false"));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index cd3ccce3331b922..bfffd3e154b3b1e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -2701,15 +2701,14 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx
         }
         olapTable.setEnableUniqueKeyMergeOnWrite(enableUniqueKeyMergeOnWrite);
 
-        boolean enableUniqueKeySkipBitmap = false;
         if (keysType == KeysType.UNIQUE_KEYS && enableUniqueKeyMergeOnWrite) {
             try {
-                enableUniqueKeySkipBitmap = PropertyAnalyzer.analyzeUniqueKeySkipBitmapColumn(properties);
+                // don't store this property, check and remove it from `properties`
+                PropertyAnalyzer.analyzeUniqueKeySkipBitmapColumn(properties);
             } catch (AnalysisException e) {
                 throw new DdlException(e.getMessage());
             }
         }
-        olapTable.setEnableUniqueKeySkipBitmap(enableUniqueKeySkipBitmap);
 
         boolean enableDeleteOnDeletePredicate = false;
         try {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
index 43b271e92a3309a..630aba4dd35a6cb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
@@ -540,7 +540,7 @@ public void validate(ConnectContext ctx) {
             if (properties != null) {
                 if (properties.containsKey(PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN)
                         && !(keysType.equals(KeysType.UNIQUE_KEYS) && isEnableMergeOnWrite)) {
-                    throw new AnalysisException("tablet property enable_unique_key_skip_bitmap_column can"
+                    throw new AnalysisException("table property enable_unique_key_skip_bitmap_column can"
                             + "only be set in merge-on-write unique table.");
                 }
                 // the merge-on-write table must have enable_unique_key_skip_bitmap_column table property
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java
index 85042fc9fdcbb81..bb2ea34bb630077 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java
@@ -368,7 +368,7 @@ public void validate(ConnectContext ctx) throws UserException {
             this.needTableStable = false;
             this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
         } else if (properties.containsKey(PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN)) {
-            // do nothing, will be analyzed when creating alter job
+            throw new AnalysisException("You can not modify property 'enable_unique_key_skip_bitmap_column'.");
         } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_PAGE_SIZE)) {
             throw new AnalysisException("You can not modify storage_page_size");
         } else {
diff --git a/regression-test/data/query_p0/system/test_table_properties.out b/regression-test/data/query_p0/system/test_table_properties.out
index 812b7d2d4b79455..70413bc48640e42 100644
--- a/regression-test/data/query_p0/system/test_table_properties.out
+++ b/regression-test/data/query_p0/system/test_table_properties.out
@@ -1,6 +1,6 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !select_check_1 --
-105
+102
 
 -- !select_check_2 --
 internal	test_table_properties_db	duplicate_table	_auto_bucket	false
@@ -16,7 +16,6 @@ internal	test_table_properties_db	duplicate_table	disable_auto_compaction	false
 internal	test_table_properties_db	duplicate_table	enable_mow_light_delete	false
 internal	test_table_properties_db	duplicate_table	enable_single_replica_compaction	false
 internal	test_table_properties_db	duplicate_table	enable_unique_key_merge_on_write	false
-internal	test_table_properties_db	duplicate_table	enable_unique_key_skip_bitmap_column	false
 internal	test_table_properties_db	duplicate_table	file_cache_ttl_seconds	0
 internal	test_table_properties_db	duplicate_table	group_commit_data_bytes	134217728
 internal	test_table_properties_db	duplicate_table	group_commit_interval_ms	10000
@@ -50,7 +49,6 @@ internal	test_table_properties_db	listtable	disable_auto_compaction	false
 internal	test_table_properties_db	listtable	enable_mow_light_delete	false
 internal	test_table_properties_db	listtable	enable_single_replica_compaction	false
 internal	test_table_properties_db	listtable	enable_unique_key_merge_on_write	false
-internal	test_table_properties_db	listtable	enable_unique_key_skip_bitmap_column	false
 internal	test_table_properties_db	listtable	file_cache_ttl_seconds	0
 internal	test_table_properties_db	listtable	group_commit_data_bytes	134217728
 internal	test_table_properties_db	listtable	group_commit_interval_ms	10000
@@ -84,7 +82,6 @@ internal	test_table_properties_db	unique_table	disable_auto_compaction	false
 internal	test_table_properties_db	unique_table	enable_mow_light_delete	false
 internal	test_table_properties_db	unique_table	enable_single_replica_compaction	false
 internal	test_table_properties_db	unique_table	enable_unique_key_merge_on_write	true
-internal	test_table_properties_db	unique_table	enable_unique_key_skip_bitmap_column	false
 internal	test_table_properties_db	unique_table	file_cache_ttl_seconds	0
 internal	test_table_properties_db	unique_table	group_commit_data_bytes	134217728
 internal	test_table_properties_db	unique_table	group_commit_interval_ms	10000
@@ -120,7 +117,6 @@ internal	test_table_properties_db	duplicate_table	disable_auto_compaction	false
 internal	test_table_properties_db	duplicate_table	enable_mow_light_delete	false
 internal	test_table_properties_db	duplicate_table	enable_single_replica_compaction	false
 internal	test_table_properties_db	duplicate_table	enable_unique_key_merge_on_write	false
-internal	test_table_properties_db	duplicate_table	enable_unique_key_skip_bitmap_column	false
 internal	test_table_properties_db	duplicate_table	file_cache_ttl_seconds	0
 internal	test_table_properties_db	duplicate_table	group_commit_data_bytes	134217728
 internal	test_table_properties_db	duplicate_table	group_commit_interval_ms	10000
@@ -154,7 +150,6 @@ internal	test_table_properties_db	unique_table	disable_auto_compaction	false
 internal	test_table_properties_db	unique_table	enable_mow_light_delete	false
 internal	test_table_properties_db	unique_table	enable_single_replica_compaction	false
 internal	test_table_properties_db	unique_table	enable_unique_key_merge_on_write	true
-internal	test_table_properties_db	unique_table	enable_unique_key_skip_bitmap_column	false
 internal	test_table_properties_db	unique_table	file_cache_ttl_seconds	0
 internal	test_table_properties_db	unique_table	group_commit_data_bytes	134217728
 internal	test_table_properties_db	unique_table	group_commit_interval_ms	10000
@@ -192,7 +187,6 @@ internal	test_table_properties_db	duplicate_table	disable_auto_compaction	false
 internal	test_table_properties_db	duplicate_table	enable_mow_light_delete	false
 internal	test_table_properties_db	duplicate_table	enable_single_replica_compaction	false
 internal	test_table_properties_db	duplicate_table	enable_unique_key_merge_on_write	false
-internal	test_table_properties_db	duplicate_table	enable_unique_key_skip_bitmap_column	false
 internal	test_table_properties_db	duplicate_table	file_cache_ttl_seconds	0
 internal	test_table_properties_db	duplicate_table	group_commit_data_bytes	134217728
 internal	test_table_properties_db	duplicate_table	group_commit_interval_ms	10000
diff --git a/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_property.groovy b/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_property.groovy
index 6b51d0a8fbcae9d..4e26a6a7de45ada 100644
--- a/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_property.groovy
+++ b/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_property.groovy
@@ -73,6 +73,11 @@ suite('test_flexible_partial_update_property') {
         }
     }
 
+    test {
+        sql """alter table ${tableName} set ("enable_unique_key_skip_bitmap_column"="true");"""
+        exception "You can not modify property 'enable_unique_key_skip_bitmap_column'."
+    }
+
     doSchemaChange """alter table ${tableName} enable feature "UPDATE_FLEXIBLE_COLUMNS";"""
     show_res = sql "show create table ${tableName}"
     assertTrue(show_res.toString().contains('"enable_unique_key_skip_bitmap_column" = "true"'))

From bcf08cd9df8448aad61a3bf5aaa3cfffc42e33c8 Mon Sep 17 00:00:00 2001
From: Socrates <suyiteng@selectdb.com>
Date: Tue, 7 Jan 2025 15:19:38 +0800
Subject: [PATCH 021/140] [fix](shellcheck) fix hive-metastore and enable
 shellcheck in docker (#46496)

Problem Summary:
This pull request includes changes to the
`.github/workflows/code-checks.yml` and
`docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh`
files. The most important changes involve updating the shell checker
exclusions and modifying the script execution behavior for the Hive
metastore.
---
 .github/workflows/code-checks.yml             |  2 +-
 .../hive/scripts/hive-metastore.sh            | 32 +++++++++----------
 2 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml
index 6aaa83f47cd4ffa..ac368fcf6a2197a 100644
--- a/.github/workflows/code-checks.yml
+++ b/.github/workflows/code-checks.yml
@@ -50,7 +50,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           sh_checker_comment: true
-          sh_checker_exclude: .git .github ^docker ^thirdparty/src ^thirdparty/installed ^ui ^docs/node_modules ^tools/clickbench-tools ^extension ^output ^fs_brokers/apache_hdfs_broker/output (^|.*/)Dockerfile$ ^be/src/apache-orc ^be/src/clucene ^pytest ^samples
+          sh_checker_exclude: .git .github ^docker/compilation ^docker/runtime ^thirdparty/src ^thirdparty/installed ^ui ^docs/node_modules ^tools/clickbench-tools ^extension ^output ^fs_brokers/apache_hdfs_broker/output (^|.*/)Dockerfile$ ^be/src/apache-orc ^be/src/clucene ^pytest ^samples
 
   preparation:
     name: "Clang Tidy Preparation"
diff --git a/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh b/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh
index b9982c2ed603edc..7ff6bc4c62f2fe6 100755
--- a/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh
+++ b/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh
@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -x
+set -e -x
 
 nohup /opt/hive/bin/hive --service metastore &
 
@@ -27,14 +27,14 @@ sleep 10s
 # new cases should use separate dir
 hadoop fs -mkdir -p /user/doris/suites/
 
-lockfile1 = "mnt/scripts/run-data.lock"
+lockfile1="/mnt/scripts/run-data.lock"
 
 # wait lockfile
-while [ -f "$lockfile1" ]; do
+while [[ -f "${lockfile1}" ]]; do
     sleep 10
 done
 
-touch "$lockfile1"
+touch "${lockfile1}"
 
 DATA_DIR="/mnt/scripts/data/"
 find "${DATA_DIR}" -type f -name "run.sh" -print0 | xargs -0 -n 1 -P 10 -I {} sh -c '
@@ -45,16 +45,16 @@ find "${DATA_DIR}" -type f -name "run.sh" -print0 | xargs -0 -n 1 -P 10 -I {} sh
     echo "Script: {} executed in $EXECUTION_TIME seconds"
 '
 
-rm -f "$lockfile1"
+rm -f "${lockfile1}"
 
-lockfile2 = "mnt/scripts/download-data.lock"
+lockfile2="/mnt/scripts/download-data.lock"
 
 # wait lockfile
-while [ -f "$lockfile2" ]; do
+while [[ -f "${lockfile2}" ]]; do
     sleep 10
 done
 
-touch "$lockfile2"
+touch "${lockfile2}"
 
 # if you test in your local，better use # to annotation section about tpch1.db
 if [[ ! -d "/mnt/scripts/tpch1.db" ]]; then
@@ -89,39 +89,39 @@ else
     echo "/mnt/scripts/tvf_data exist, continue !"
 fi
 
-rm -f "$lockfile2"
+rm -f "${lockfile2}"
 
 # put data file
 ## put tpch1
-if [ -z "$(ls /mnt/scripts/tpch1.db)" ]; then
+if [[ -z "$(ls /mnt/scripts/tpch1.db)" ]]; then
     echo "tpch1.db does not exist"
     exit 1
 fi
 hadoop fs -mkdir -p /user/doris/
 hadoop fs -put /mnt/scripts/tpch1.db /user/doris/
-if [ -z "$(hadoop fs -ls /user/doris/tpch1.db)" ]; then
+if [[ -z "$(hadoop fs -ls /user/doris/tpch1.db)" ]]; then
     echo "tpch1.db put failed"
     exit 1
 fi
 
 ## put paimon1
-if [ -z "$(ls /mnt/scripts/paimon1)" ]; then
+if [[ -z "$(ls /mnt/scripts/paimon1)" ]]; then
     echo "paimon1 does not exist"
     exit 1
 fi
 hadoop fs -put /mnt/scripts/paimon1 /user/doris/
-if [ -z "$(hadoop fs -ls /user/doris/paimon1)" ]; then
+if [[ -z "$(hadoop fs -ls /user/doris/paimon1)" ]]; then
     echo "paimon1 put failed"
     exit 1
 fi
 
 ## put tvf_data
-if [ -z "$(ls /mnt/scripts/tvf_data)" ]; then
+if [[ -z "$(ls /mnt/scripts/tvf_data)" ]]; then
     echo "tvf_data does not exist"
     exit 1
 fi
 hadoop fs -put /mnt/scripts/tvf_data /user/doris/
-if [ -z "$(hadoop fs -ls /user/doris/tvf_data)" ]; then
+if [[ -z "$(hadoop fs -ls /user/doris/tvf_data)" ]]; then
     echo "tvf_data put failed"
     exit 1
 fi
@@ -143,7 +143,7 @@ START_TIME=$(date +%s)
 hive -f /mnt/scripts/create_view_scripts/create_view.hql
 END_TIME=$(date +%s)
 EXECUTION_TIME=$((END_TIME - START_TIME))
-echo "Script: create_view.hql executed in $EXECUTION_TIME seconds"
+echo "Script: create_view.hql executed in ${EXECUTION_TIME} seconds"
 
 touch /mnt/SUCCESS
 

From b110bc03bcce0b06e149e36649254254d90c1744 Mon Sep 17 00:00:00 2001
From: chunping <cece_mail@163.com>
Date: Tue, 7 Jan 2025 15:45:22 +0800
Subject: [PATCH 022/140] (fix)[test] avoid use gloabal define in cache
 load.groovy (#46532)

### What problem does this PR solve?

Issue Number: close #xxx

Related PR: #xxx

Problem Summary: load.groovy exec fail because use global define, which
make other cases failed
---
 regression-test/suites/cloud_p0/cache/load.groovy | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/regression-test/suites/cloud_p0/cache/load.groovy b/regression-test/suites/cloud_p0/cache/load.groovy
index 8dc855414261797..9485d16e22dd50e 100644
--- a/regression-test/suites/cloud_p0/cache/load.groovy
+++ b/regression-test/suites/cloud_p0/cache/load.groovy
@@ -52,7 +52,7 @@ suite("load") {
     println("the brpc port is " + brpcPortList);
 
     for (unique_id : beUniqueIdList) {
-        resp = get_cluster.call(unique_id);
+        def resp = get_cluster.call(unique_id);
         for (cluster : resp) {
             if (cluster.type == "COMPUTE") {
                 drop_cluster.call(cluster.cluster_name, cluster.cluster_id);
@@ -70,10 +70,10 @@ suite("load") {
                      "regression_cluster_name1", "regression_cluster_id1");
     sleep(20000)
 
-    result  = sql "show clusters"
+    def new_res  = sql "show clusters"
     sql """ SET PROPERTY 'default_cloud_cluster' = "regression_cluster_name0"; """
-    assertEquals(result.size(), 2);
+    assertEquals(new_res.size(), 2);
     sql """ set global enable_auto_analyze = false; """
     sql """ drop table if exists __internal_schema.column_statistics; """
     sql """ drop table if exists __internal_schema.histogram_statistics; """
-}
\ No newline at end of file
+}

From 5e8105b9dd6ea0c00b7595150759505e68f9bcd7 Mon Sep 17 00:00:00 2001
From: morrySnow <zhangwenxin@selectdb.com>
Date: Tue, 7 Jan 2025 15:57:40 +0800
Subject: [PATCH 023/140] [chore](MTMV) catch all exception when get mtmvs to
 avoid plan failed (#46474)

### What problem does this PR solve?

Related PR: #28144 #45045

Problem Summary:
if getTable return Table is not MTMV, ClassCastException raised and lead
to plan failed.
---
 .../main/java/org/apache/doris/mtmv/MTMVRelationManager.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
index f8f92e25d38d654..f3939cb47eafd8d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
@@ -90,7 +90,7 @@ public Set<MTMV> getAvailableMTMVs(List<BaseTableInfo> tableInfos, ConnectContex
                 if (isMVPartitionValid(mtmv, ctx, forceConsistent)) {
                     res.add(mtmv);
                 }
-            } catch (AnalysisException e) {
+            } catch (Exception e) {
                 // not throw exception to client, just ignore it
                 LOG.warn("getTable failed: {}", tableInfo.toString(), e);
             }
@@ -107,7 +107,7 @@ public Set<MTMV> getAllMTMVs(List<BaseTableInfo> tableInfos) {
         for (BaseTableInfo tableInfo : mvInfos) {
             try {
                 mtmvs.add((MTMV) MTMVUtil.getTable(tableInfo));
-            } catch (AnalysisException e) {
+            } catch (Exception e) {
                 // not throw exception to client, just ignore it
                 LOG.warn("getTable failed: {}", tableInfo.toString(), e);
             }

From bdab3bd4d7268c22fa0531788c8b911b1fd93656 Mon Sep 17 00:00:00 2001
From: lihangyu <lihangyu@selectdb.com>
Date: Tue, 7 Jan 2025 16:10:51 +0800
Subject: [PATCH 024/140] [Fix](Variant) variant should not implicit be short
 key column when create mv (#46444)

When creating mv, the variant column should not be part of short key
since it's not supported sorting
---
 .../java/org/apache/doris/catalog/Type.java   | 12 +++++++
 .../doris/alter/MaterializedViewHandler.java  |  3 +-
 .../analysis/CreateMaterializedViewStmt.java  |  6 ++--
 .../doris/analysis/CreateTableStmt.java       | 11 +------
 .../java/org/apache/doris/catalog/Env.java    |  2 +-
 .../plans/commands/info/CreateMTMVInfo.java   |  8 ++---
 .../plans/commands/info/CreateTableInfo.java  |  3 +-
 .../data/variant_p0/mv/multi_slot.out         |  3 ++
 .../test_create_mv_complex_type.groovy        |  8 ++---
 .../test_materialized_view_array.groovy       |  2 +-
 .../test_materialized_view_struct.groovy      |  2 +-
 .../suites/variant_p0/mv/multi_slot.groovy    | 31 +++++++++++++++++--
 12 files changed, 62 insertions(+), 29 deletions(-)

diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
index 7dfcfd15ebec84b..d3218203bf6462a 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
@@ -382,6 +382,18 @@ public boolean supportSubType(Type subType) {
         return false;
     }
 
+    /**
+    * Return true if this type can be as short key
+    */
+    public boolean couldBeShortKey() {
+        return !(isFloatingPointType()
+                        || getPrimitiveType() == PrimitiveType.STRING
+                        || isJsonbType()
+                        || isComplexType()
+                        || isObjectStored()
+                        || isVariantType());
+    }
+
     /**
      * The output of this is stored directly in the hive metastore as the column type.
      * The string must match exactly.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
index a6f1cae99876782..64a747e99e4d6a5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
@@ -795,10 +795,9 @@ public List<Column> checkAndPrepareMaterializedView(AddRollupClause addRollupCla
                             break;
                         }
                     }
-                    if (column.getType().isFloatingPointType()) {
+                    if (!column.getType().couldBeShortKey()) {
                         break;
                     }
-
                     column.setIsKey(true);
 
                     if (column.getType().getPrimitiveType() == PrimitiveType.VARCHAR) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
index 68ec82387020ab5..2922595fbce00eb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
@@ -476,7 +476,7 @@ private void supplyOrderColumn() throws AnalysisException {
                     }
                     break;
                 }
-                if (column.getType().isFloatingPointType()) {
+                if (!column.getType().couldBeShortKey()) {
                     break;
                 }
                 if (column.getType().getPrimitiveType() == PrimitiveType.VARCHAR) {
@@ -487,7 +487,9 @@ private void supplyOrderColumn() throws AnalysisException {
                 column.setIsKey(true);
             }
             if (theBeginIndexOfValue == 0) {
-                throw new AnalysisException("The first column could not be float or double type, use decimal instead");
+                throw new AnalysisException(
+                    "The first column could not be float, double or complex "
+                    + "type like array, struct, map, json, variant.");
             }
             // supply value
             for (; theBeginIndexOfValue < mvColumnItemList.size(); theBeginIndexOfValue++) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
index 5f173b4a243702a..9232d66050a30e6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
@@ -375,16 +375,7 @@ public void analyze(Analyzer analyzer) throws UserException {
                                 }
                                 break;
                             }
-                            if (columnDef.getType().isFloatingPointType()) {
-                                break;
-                            }
-                            if (columnDef.getType().getPrimitiveType() == PrimitiveType.STRING) {
-                                break;
-                            }
-                            if (columnDef.getType().getPrimitiveType() == PrimitiveType.JSONB) {
-                                break;
-                            }
-                            if (columnDef.getType().isComplexType()) {
+                            if (!columnDef.getType().couldBeShortKey()) {
                                 break;
                             }
                             if (columnDef.getType().getPrimitiveType() == PrimitiveType.VARCHAR) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 19865446456967c..505970e104c26f6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -4699,7 +4699,7 @@ public static short calcShortKeyColumnCount(List<Column> columns, Map<String, St
                     }
                     break;
                 }
-                if (column.getType().isFloatingPointType()) {
+                if (!column.getType().couldBeShortKey()) {
                     break;
                 }
                 if (column.getDataType() == PrimitiveType.VARCHAR) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
index bab4eb7a3142f8a..41a0de198ac2a6d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
@@ -314,10 +314,10 @@ private void analyzeKeys() {
                     }
                     break;
                 }
-                if (type.isFloatLikeType() || type.isStringType() || type.isJsonType()
-                        || catalogType.isComplexType() || type.isBitmapType() || type.isHllType()
-                        || type.isQuantileStateType() || type.isJsonType() || type.isStructType()
-                        || column.getAggType() != null || type.isVariantType()) {
+                if (column.getAggType() != null) {
+                    break;
+                }
+                if (!catalogType.couldBeShortKey()) {
                     break;
                 }
                 keys.add(column.getName());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
index 630aba4dd35a6cb..c43fe04fb2c1f16 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
@@ -382,8 +382,7 @@ public void validate(ConnectContext ctx) {
                                 }
                                 break;
                             }
-                            if (type.isFloatLikeType() || type.isStringType() || type.isJsonType()
-                                    || catalogType.isComplexType() || catalogType.isVariantType()) {
+                            if (!catalogType.couldBeShortKey()) {
                                 break;
                             }
                             keys.add(column.getName());
diff --git a/regression-test/data/variant_p0/mv/multi_slot.out b/regression-test/data/variant_p0/mv/multi_slot.out
index e146ef05e56a16e..259214967d3f854 100644
--- a/regression-test/data/variant_p0/mv/multi_slot.out
+++ b/regression-test/data/variant_p0/mv/multi_slot.out
@@ -26,3 +26,6 @@
 456	\N
 789	\N
 
+-- !sql --
+2021-01-01T11:11:11	"http://xxx.xxx.xxx"	12
+
diff --git a/regression-test/suites/mv_p0/test_create_mv_complex_type/test_create_mv_complex_type.groovy b/regression-test/suites/mv_p0/test_create_mv_complex_type/test_create_mv_complex_type.groovy
index 1e07d69ba828712..2ce17abcc0be0fa 100644
--- a/regression-test/suites/mv_p0/test_create_mv_complex_type/test_create_mv_complex_type.groovy
+++ b/regression-test/suites/mv_p0/test_create_mv_complex_type/test_create_mv_complex_type.groovy
@@ -47,7 +47,7 @@ suite ("create_mv_complex_type") {
         sql """create materialized view mv as select c_jsonb, c_int from base_table;"""
         success = true
     } catch (Exception e) {
-        assertTrue(e.getMessage().contains("not support to create materialized view"), e.getMessage())
+        assertTrue(e.getMessage().contains("The first column could not be"), e.getMessage())
     }
     assertFalse(success)
 
@@ -65,7 +65,7 @@ suite ("create_mv_complex_type") {
         sql """create materialized view mv as select c_array, c_int from base_table;"""
         success = true
     } catch (Exception e) {
-        assertTrue(e.getMessage().contains("not support to create materialized view"), e.getMessage())
+        assertTrue(e.getMessage().contains("The first column could not be"), e.getMessage())
     }
     assertFalse(success)
 
@@ -83,7 +83,7 @@ suite ("create_mv_complex_type") {
         sql """create materialized view mv as select c_map, c_int from base_table;"""
         success = true
     } catch (Exception e) {
-        assertTrue(e.getMessage().contains("not support to create materialized view"), e.getMessage())
+        assertTrue(e.getMessage().contains("The first column could not be"), e.getMessage())
     }
     assertFalse(success)
 
@@ -101,7 +101,7 @@ suite ("create_mv_complex_type") {
         sql """create materialized view mv as select c_struct, c_int from base_table;"""
         success = true
     } catch (Exception e) {
-        assertTrue(e.getMessage().contains("not support to create materialized view"), e.getMessage())
+        assertTrue(e.getMessage().contains("The first column could not be"), e.getMessage())
     }
     assertFalse(success)
 
diff --git a/regression-test/suites/rollup_p0/test_materialized_view_array.groovy b/regression-test/suites/rollup_p0/test_materialized_view_array.groovy
index dad735a76ce61e6..52b8a32740d0c0a 100644
--- a/regression-test/suites/rollup_p0/test_materialized_view_array.groovy
+++ b/regression-test/suites/rollup_p0/test_materialized_view_array.groovy
@@ -67,7 +67,7 @@ suite("test_materialized_view_array", "rollup") {
         create_test_table.call(tableName)
         test {
             sql "CREATE MATERIALIZED VIEW idx AS select k2,k1, k3, k4, k5 from ${tableName}"
-            exception "errCode = 2, detailMessage = The ARRAY column[`mv_k2` array<smallint> NULL] not support to create materialized view"
+            exception "errCode = 2, detailMessage = The first column could not be float, double or complex type like array, struct, map, json, variant"
         }
     } finally {
         try_sql("DROP TABLE IF EXISTS ${tableName}")
diff --git a/regression-test/suites/rollup_p0/test_materialized_view_struct.groovy b/regression-test/suites/rollup_p0/test_materialized_view_struct.groovy
index 61e8415cacc68b8..2a05b89538433f7 100644
--- a/regression-test/suites/rollup_p0/test_materialized_view_struct.groovy
+++ b/regression-test/suites/rollup_p0/test_materialized_view_struct.groovy
@@ -59,7 +59,7 @@ suite("test_materialized_view_struct", "rollup") {
         create_test_table.call(tableName)
         test {
             sql "CREATE MATERIALIZED VIEW idx AS select k2,k1, k3, k4, k5 from ${tableName}"
-            exception "errCode = 2, detailMessage = The STRUCT column[`mv_k2` struct<f1:smallint> NULL] not support to create materialized view"
+            exception "errCode = 2, detailMessage = The first column could not be float, double or complex type like array, struct, map, json, variant."
         }
     } finally {
         try_sql("DROP TABLE IF EXISTS ${tableName}")
diff --git a/regression-test/suites/variant_p0/mv/multi_slot.groovy b/regression-test/suites/variant_p0/mv/multi_slot.groovy
index fd1727c571df556..1490c30e58778be 100644
--- a/regression-test/suites/variant_p0/mv/multi_slot.groovy
+++ b/regression-test/suites/variant_p0/mv/multi_slot.groovy
@@ -50,8 +50,35 @@ suite ("multi_slot") {
 
     order_qt_select_star "select abs(cast(v['k1'] as int))+cast(v['k2'] as int)+1,abs(cast(v['k2'] as int)+2)+cast(v['k3'] as int)+3 from multi_slot;"
     order_qt_select_star "select * from multi_slot order by cast(v['k1'] as int);"
-    // TODO fix and remove enable_rewrite_element_at_to_slot
-    order_qt_select_star "select /*+SET_VAR(enable_rewrite_element_at_to_slot=false) */ abs(cast(v['k4']['k44'] as int)), sum(abs(cast(v['k2'] as int)+2)+cast(v['k3'] as int)+3) from multi_slot group by abs(cast(v['k4']['k44'] as int))"
+    order_qt_select_star "select abs(cast(v['k4']['k44'] as int)), sum(abs(cast(v['k2'] as int)+2)+cast(v['k3'] as int)+3) from multi_slot group by abs(cast(v['k4']['k44'] as int))"
+
+    sql "drop table if exists test_mv"
+    sql """
+    CREATE TABLE `test_mv` (
+        `handle_time` datetime NOT NULL ,
+        `client_request` variant NULL,
+        `status` int NULL
+    )  
+    DISTRIBUTED BY HASH(`handle_time`)
+    BUCKETS 10 PROPERTIES (
+      "is_being_synced" = "false",
+      "storage_medium" = "hdd",
+      "storage_format" = "V2",
+      "inverted_index_storage_format" = "V1",
+      "light_schema_change" = "true",
+      "disable_auto_compaction" = "false",
+      "enable_single_replica_compaction" = "false",
+      "replication_num" = "1"
+    );  
+    """
+    sql """insert into test_mv values ('2021-01-01 11:11:11', '{"url" : "http://xxx.xxx.xxx"}', 12)"""
+    createMV("create materialized view mv_1 as select `handle_time`, `client_request`['url'] as `uri`, `status` from test_mv")
+    qt_sql "select `handle_time`, `client_request`['url'] as `uri`, `status` from test_mv"
+    test {
+        sql "create materialized view mv_x as select `client_request`['url'] as `uri`, `status` from test_mv"
+        exception("The first column could not be float, double or complex type like array, struct, map, json, variant.")
+    }
+
 
     // def retry_times = 60
     // for (def i = 0; i < retry_times; ++i) {

From 95d7d88c34c250fba8667695296f0421ad7e5e48 Mon Sep 17 00:00:00 2001
From: yiguolei <guolei@selectdb.com>
Date: Tue, 7 Jan 2025 19:03:49 +0800
Subject: [PATCH 025/140] [chore](thrift) remove some useless thrift
 definitions (#46492)

---
 be/src/common/config.h                        |   2 +-
 be/src/exec/schema_scanner/schema_helper.cpp  |   2 -
 be/src/exec/schema_scanner/schema_helper.h    |   2 -
 be/src/pipeline/exec/schema_scan_operator.cpp |   8 -
 be/src/service/backend_service.cpp            |  41 --
 be/src/service/backend_service.h              |   2 -
 be/src/service/internal_service.cpp           |  15 -
 be/src/service/internal_service.h             |  15 -
 be/src/util/debug_util.h                      |   1 -
 .../java/org/apache/doris/qe/Coordinator.java |   1 -
 .../doris/service/FrontendServiceImpl.java    |  70 ----
 .../apache/doris/common/GenericPoolTest.java  |   7 -
 .../doris/utframe/MockedBackendFactory.java   |  14 -
 gensrc/proto/internal_service.proto           |   2 -
 gensrc/thrift/BackendService.thrift           |   5 -
 gensrc/thrift/Data.thrift                     |  26 --
 gensrc/thrift/Ddl.thrift                      | 365 ------------------
 gensrc/thrift/FrontendService.thrift          | 218 -----------
 gensrc/thrift/PaloInternalService.thrift      |  69 ----
 gensrc/thrift/PaloService.thrift              | 100 -----
 gensrc/thrift/PlanNodes.thrift                |  15 -
 gensrc/thrift/Types.thrift                    |   7 -
 22 files changed, 1 insertion(+), 986 deletions(-)
 delete mode 100644 gensrc/thrift/Ddl.thrift
 delete mode 100644 gensrc/thrift/PaloService.thrift

diff --git a/be/src/common/config.h b/be/src/common/config.h
index ddca52c607b495e..6c84d89fabac502 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -586,7 +586,7 @@ DECLARE_mInt64(load_error_log_limit_bytes);
 
 // be brpc interface is classified into two categories: light and heavy
 // each category has diffrent thread number
-// threads to handle heavy api interface, such as transmit_data/transmit_block etc
+// threads to handle heavy api interface, such as transmit_block etc
 DECLARE_Int32(brpc_heavy_work_pool_threads);
 // threads to handle light api interface, such as exec_plan_fragment_prepare/exec_plan_fragment_start
 DECLARE_Int32(brpc_light_work_pool_threads);
diff --git a/be/src/exec/schema_scanner/schema_helper.cpp b/be/src/exec/schema_scanner/schema_helper.cpp
index 2819dc603f70d06..7cf95187b02423f 100644
--- a/be/src/exec/schema_scanner/schema_helper.cpp
+++ b/be/src/exec/schema_scanner/schema_helper.cpp
@@ -23,8 +23,6 @@
 #include "util/thrift_rpc_helper.h"
 
 namespace doris {
-class TDescribeTableParams;
-class TDescribeTableResult;
 class TDescribeTablesParams;
 class TDescribeTablesResult;
 class TGetDbsParams;
diff --git a/be/src/exec/schema_scanner/schema_helper.h b/be/src/exec/schema_scanner/schema_helper.h
index 752e282bb52f2ae..bc79409312813d5 100644
--- a/be/src/exec/schema_scanner/schema_helper.h
+++ b/be/src/exec/schema_scanner/schema_helper.h
@@ -24,8 +24,6 @@
 #include "common/status.h"
 
 namespace doris {
-class TDescribeTableParams;
-class TDescribeTableResult;
 class TDescribeTablesParams;
 class TDescribeTablesResult;
 class TGetDbsParams;
diff --git a/be/src/pipeline/exec/schema_scan_operator.cpp b/be/src/pipeline/exec/schema_scan_operator.cpp
index 2e2f80f5e248387..ea6f122df95616e 100644
--- a/be/src/pipeline/exec/schema_scan_operator.cpp
+++ b/be/src/pipeline/exec/schema_scan_operator.cpp
@@ -192,14 +192,6 @@ Status SchemaScanOperatorX::open(RuntimeState* state) {
 
     _tuple_idx = 0;
 
-    if (_common_scanner_param->user) {
-        TSetSessionParams param;
-        param.__set_user(*_common_scanner_param->user);
-        //TStatus t_status;
-        //RETURN_IF_ERROR(SchemaJniHelper::set_session(param, &t_status));
-        //RETURN_IF_ERROR(Status(t_status));
-    }
-
     return Status::OK();
 }
 
diff --git a/be/src/service/backend_service.cpp b/be/src/service/backend_service.cpp
index 55e18b4deb85444..29504fef1873c2b 100644
--- a/be/src/service/backend_service.cpp
+++ b/be/src/service/backend_service.cpp
@@ -654,47 +654,6 @@ Status BaseBackendService::start_plan_fragment_execution(
                                                          QuerySource::INTERNAL_FRONTEND);
 }
 
-void BaseBackendService::transmit_data(TTransmitDataResult& return_val,
-                                       const TTransmitDataParams& params) {
-    VLOG_ROW << "transmit_data(): instance_id=" << params.dest_fragment_instance_id
-             << " node_id=" << params.dest_node_id << " #rows=" << params.row_batch.num_rows
-             << " eos=" << (params.eos ? "true" : "false");
-    // VLOG_ROW << "transmit_data params: " << apache::thrift::ThriftDebugString(params).c_str();
-
-    if (params.__isset.packet_seq) {
-        return_val.__set_packet_seq(params.packet_seq);
-        return_val.__set_dest_fragment_instance_id(params.dest_fragment_instance_id);
-        return_val.__set_dest_node_id(params.dest_node_id);
-    }
-
-    // TODO: fix Thrift so we can simply take ownership of thrift_batch instead
-    // of having to copy its data
-    if (params.row_batch.num_rows > 0) {
-        // Status status = _exec_env->stream_mgr()->add_data(
-        //         params.dest_fragment_instance_id,
-        //         params.dest_node_id,
-        //         params.row_batch,
-        //         params.sender_id);
-        // status.set_t_status(&return_val);
-
-        // if (!status.ok()) {
-        //     // should we close the channel here as well?
-        //     return;
-        // }
-    }
-
-    if (params.eos) {
-        // Status status = _exec_env->stream_mgr()->close_sender(
-        //        params.dest_fragment_instance_id,
-        //        params.dest_node_id,
-        //        params.sender_id,
-        //        params.be_number);
-        //VLOG_ROW << "params.eos: " << (params.eos ? "true" : "false")
-        //        << " close_sender status: " << status;
-        //status.set_t_status(&return_val);
-    }
-}
-
 void BaseBackendService::submit_export_task(TStatus& t_status, const TExportTaskRequest& request) {
     //    VLOG_ROW << "submit_export_task. request  is "
     //            << apache::thrift::ThriftDebugString(request).c_str();
diff --git a/be/src/service/backend_service.h b/be/src/service/backend_service.h
index 1d4219e21917b8e..d987164359751e9 100644
--- a/be/src/service/backend_service.h
+++ b/be/src/service/backend_service.h
@@ -92,8 +92,6 @@ class BaseBackendService : public BackendServiceIf {
     void cancel_plan_fragment(TCancelPlanFragmentResult& return_val,
                               const TCancelPlanFragmentParams& params) override {};
 
-    void transmit_data(TTransmitDataResult& return_val, const TTransmitDataParams& params) override;
-
     void submit_export_task(TStatus& t_status, const TExportTaskRequest& request) override;
 
     void get_export_status(TExportStatusResult& result, const TUniqueId& task_id) override;
diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp
index fb0b2f090bc0451..3186d4914bcf5d2 100644
--- a/be/src/service/internal_service.cpp
+++ b/be/src/service/internal_service.cpp
@@ -272,21 +272,6 @@ PInternalService::~PInternalService() {
     CHECK_EQ(0, bthread_key_delete(AsyncIO::btls_io_ctx_key));
 }
 
-void PInternalService::transmit_data(google::protobuf::RpcController* controller,
-                                     const PTransmitDataParams* request,
-                                     PTransmitDataResult* response,
-                                     google::protobuf::Closure* done) {}
-
-void PInternalService::transmit_data_by_http(google::protobuf::RpcController* controller,
-                                             const PEmptyRequest* request,
-                                             PTransmitDataResult* response,
-                                             google::protobuf::Closure* done) {}
-
-void PInternalService::_transmit_data(google::protobuf::RpcController* controller,
-                                      const PTransmitDataParams* request,
-                                      PTransmitDataResult* response,
-                                      google::protobuf::Closure* done, const Status& extract_st) {}
-
 void PInternalService::tablet_writer_open(google::protobuf::RpcController* controller,
                                           const PTabletWriterOpenRequest* request,
                                           PTabletWriterOpenResult* response,
diff --git a/be/src/service/internal_service.h b/be/src/service/internal_service.h
index 66a0f867393793f..e3d03a6a4494991 100644
--- a/be/src/service/internal_service.h
+++ b/be/src/service/internal_service.h
@@ -64,16 +64,6 @@ class PInternalService : public PBackendService {
     PInternalService(ExecEnv* exec_env);
     ~PInternalService() override;
 
-    void transmit_data(::google::protobuf::RpcController* controller,
-                       const ::doris::PTransmitDataParams* request,
-                       ::doris::PTransmitDataResult* response,
-                       ::google::protobuf::Closure* done) override;
-
-    void transmit_data_by_http(::google::protobuf::RpcController* controller,
-                               const ::doris::PEmptyRequest* request,
-                               ::doris::PTransmitDataResult* response,
-                               ::google::protobuf::Closure* done) override;
-
     void exec_plan_fragment(google::protobuf::RpcController* controller,
                             const PExecPlanFragmentRequest* request,
                             PExecPlanFragmentResult* result,
@@ -253,11 +243,6 @@ class PInternalService : public PBackendService {
 
     Status _fold_constant_expr(const std::string& ser_request, PConstantExprResult* response);
 
-    void _transmit_data(::google::protobuf::RpcController* controller,
-                        const ::doris::PTransmitDataParams* request,
-                        ::doris::PTransmitDataResult* response, ::google::protobuf::Closure* done,
-                        const Status& extract_st);
-
     void _transmit_block(::google::protobuf::RpcController* controller,
                          const ::doris::PTransmitDataParams* request,
                          ::doris::PTransmitDataResult* response, ::google::protobuf::Closure* done,
diff --git a/be/src/util/debug_util.h b/be/src/util/debug_util.h
index 31cc1f8f5cadb1e..19485ca07d270e7 100644
--- a/be/src/util/debug_util.h
+++ b/be/src/util/debug_util.h
@@ -27,7 +27,6 @@
 namespace doris {
 
 std::string print_plan_node_type(const TPlanNodeType::type& type);
-std::string print_tstmt_type(const TStmtType::type& type);
 std::string print_query_state(const QueryState::type& type);
 std::string PrintTUnit(const TUnit::type& type);
 std::string PrintTMetricKind(const TMetricKind::type& type);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index 472f2462e4d39db..3bf5c44d564065f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -188,7 +188,6 @@ public class Coordinator implements CoordInterface {
 
     protected ImmutableMap<Long, Backend> idToBackend = ImmutableMap.of();
 
-    // copied from TQueryExecRequest; constant across all fragments
     private final TDescriptorTable descTable;
     private FragmentIdMapping<DistributedPlan> distributedPlans;
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index f4f10bf331d1c60..8f2c37602e96539 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -140,8 +140,6 @@
 import org.apache.doris.thrift.TConfirmUnusedRemoteFilesResult;
 import org.apache.doris.thrift.TCreatePartitionRequest;
 import org.apache.doris.thrift.TCreatePartitionResult;
-import org.apache.doris.thrift.TDescribeTableParams;
-import org.apache.doris.thrift.TDescribeTableResult;
 import org.apache.doris.thrift.TDescribeTablesParams;
 import org.apache.doris.thrift.TDescribeTablesResult;
 import org.apache.doris.thrift.TDropPlsqlPackageRequest;
@@ -824,74 +822,6 @@ public TFeResult updateExportTaskStatus(TUpdateExportTaskStatusRequest request)
         return result;
     }
 
-    @Override
-    public TDescribeTableResult describeTable(TDescribeTableParams params) throws TException {
-        if (LOG.isDebugEnabled()) {
-            LOG.debug("get desc table request: {}", params);
-        }
-        TDescribeTableResult result = new TDescribeTableResult();
-        List<TColumnDef> columns = Lists.newArrayList();
-        result.setColumns(columns);
-
-        // database privs should be checked in analysis phrase
-        UserIdentity currentUser = null;
-        if (params.isSetCurrentUserIdent()) {
-            currentUser = UserIdentity.fromThrift(params.current_user_ident);
-        } else {
-            currentUser = UserIdentity.createAnalyzedUserIdentWithIp(params.user, params.user_ip);
-        }
-        String dbName = getDbNameFromMysqlTableSchema(params.catalog, params.db);
-        if (!Env.getCurrentEnv().getAccessManager()
-                .checkTblPriv(currentUser, params.catalog, dbName, params.getTableName(), PrivPredicate.SHOW)) {
-            return result;
-        }
-
-        String catalogName = Strings.isNullOrEmpty(params.catalog) ? InternalCatalog.INTERNAL_CATALOG_NAME
-                : params.catalog;
-        DatabaseIf<TableIf> db = Env.getCurrentEnv().getCatalogMgr()
-                .getCatalogOrException(catalogName, catalog -> new TException("Unknown catalog " + catalog))
-                .getDbNullable(dbName);
-        if (db != null) {
-            TableIf table = db.getTableNullableIfException(params.getTableName());
-            if (table != null) {
-                table.readLock();
-                try {
-                    List<Column> baseSchema = table.getBaseSchemaOrEmpty();
-                    for (Column column : baseSchema) {
-                        final TColumnDesc desc = new TColumnDesc(column.getName(), column.getDataType().toThrift());
-                        final Integer precision = column.getOriginType().getPrecision();
-                        if (precision != null) {
-                            desc.setColumnPrecision(precision);
-                        }
-                        final Integer columnLength = column.getOriginType().getColumnSize();
-                        if (columnLength != null) {
-                            desc.setColumnLength(columnLength);
-                        }
-                        final Integer decimalDigits = column.getOriginType().getDecimalDigits();
-                        if (decimalDigits != null) {
-                            desc.setColumnScale(decimalDigits);
-                        }
-                        desc.setIsAllowNull(column.isAllowNull());
-                        final TColumnDef colDef = new TColumnDef(desc);
-                        final String comment = column.getComment();
-                        if (comment != null) {
-                            colDef.setComment(comment);
-                        }
-                        if (column.isKey()) {
-                            if (table instanceof OlapTable) {
-                                desc.setColumnKey(((OlapTable) table).getKeysType().toMetadata());
-                            }
-                        }
-                        columns.add(colDef);
-                    }
-                } finally {
-                    table.readUnlock();
-                }
-            }
-        }
-        return result;
-    }
-
     @Override
     public TDescribeTablesResult describeTables(TDescribeTablesParams params) throws TException {
         if (LOG.isDebugEnabled()) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/GenericPoolTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/GenericPoolTest.java
index 31fffe6a3327889..2885dc12caf5b8e 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/common/GenericPoolTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/GenericPoolTest.java
@@ -55,8 +55,6 @@
 import org.apache.doris.thrift.TSyncLoadForTabletsRequest;
 import org.apache.doris.thrift.TSyncLoadForTabletsResponse;
 import org.apache.doris.thrift.TTabletStatResult;
-import org.apache.doris.thrift.TTransmitDataParams;
-import org.apache.doris.thrift.TTransmitDataResult;
 import org.apache.doris.thrift.TUniqueId;
 import org.apache.doris.thrift.TWarmUpCacheAsyncRequest;
 import org.apache.doris.thrift.TWarmUpCacheAsyncResponse;
@@ -135,11 +133,6 @@ public TCancelPlanFragmentResult cancelPlanFragment(TCancelPlanFragmentParams pa
             return new TCancelPlanFragmentResult();
         }
 
-        @Override
-        public TTransmitDataResult transmitData(TTransmitDataParams params) {
-            return new TTransmitDataResult();
-        }
-
         @Override
         public TAgentResult submitTasks(List<TAgentTaskRequest> tasks) throws TException {
             return null;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java
index 1a9a175366e528b..680b3e3641e812a 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java
@@ -80,8 +80,6 @@
 import org.apache.doris.thrift.TTabletInfo;
 import org.apache.doris.thrift.TTabletStatResult;
 import org.apache.doris.thrift.TTaskType;
-import org.apache.doris.thrift.TTransmitDataParams;
-import org.apache.doris.thrift.TTransmitDataResult;
 import org.apache.doris.thrift.TUniqueId;
 import org.apache.doris.thrift.TWarmUpCacheAsyncRequest;
 import org.apache.doris.thrift.TWarmUpCacheAsyncResponse;
@@ -366,11 +364,6 @@ public TCancelPlanFragmentResult cancelPlanFragment(TCancelPlanFragmentParams pa
             return null;
         }
 
-        @Override
-        public TTransmitDataResult transmitData(TTransmitDataParams params) throws TException {
-            return null;
-        }
-
         @Override
         public TAgentResult submitTasks(List<TAgentTaskRequest> tasks) throws TException {
             for (TAgentTaskRequest request : tasks) {
@@ -507,13 +500,6 @@ public TGetRealtimeExecStatusResponse getRealtimeExecStatus(TGetRealtimeExecStat
 
     // The default Brpc service.
     public static class DefaultPBackendServiceImpl extends PBackendServiceGrpc.PBackendServiceImplBase {
-        @Override
-        public void transmitData(InternalService.PTransmitDataParams request,
-                                 StreamObserver<InternalService.PTransmitDataResult> responseObserver) {
-            responseObserver.onNext(InternalService.PTransmitDataResult.newBuilder()
-                    .setStatus(Types.PStatus.newBuilder().setStatusCode(0)).build());
-            responseObserver.onCompleted();
-        }
 
         @Override
         public void execPlanFragment(InternalService.PExecPlanFragmentRequest request,
diff --git a/gensrc/proto/internal_service.proto b/gensrc/proto/internal_service.proto
index 547b2588168755e..837d3f4a941e334 100644
--- a/gensrc/proto/internal_service.proto
+++ b/gensrc/proto/internal_service.proto
@@ -990,8 +990,6 @@ message PGetBeResourceResponse {
 }
 
 service PBackendService {
-    rpc transmit_data(PTransmitDataParams) returns (PTransmitDataResult);
-    rpc transmit_data_by_http(PEmptyRequest) returns (PTransmitDataResult);
     // If #fragments of a query is < 3, use exec_plan_fragment directly.
     // If #fragments of a query is >=3, use exec_plan_fragment_prepare + exec_plan_fragment_start
     rpc exec_plan_fragment(PExecPlanFragmentRequest) returns (PExecPlanFragmentResult);
diff --git a/gensrc/thrift/BackendService.thrift b/gensrc/thrift/BackendService.thrift
index aed248adfe891a3..3c2f256bac9e863 100644
--- a/gensrc/thrift/BackendService.thrift
+++ b/gensrc/thrift/BackendService.thrift
@@ -357,11 +357,6 @@ service BackendService {
     PaloInternalService.TCancelPlanFragmentResult cancel_plan_fragment(
         1:PaloInternalService.TCancelPlanFragmentParams params);
 
-    // Called by sender to transmit single row batch. Returns error indication
-    // if params.fragmentId or params.destNodeId are unknown or if data couldn't be read.
-    PaloInternalService.TTransmitDataResult transmit_data(
-        1:PaloInternalService.TTransmitDataParams params);
-
     AgentService.TAgentResult submit_tasks(1:list<AgentService.TAgentTaskRequest> tasks);
 
     AgentService.TAgentResult make_snapshot(1:AgentService.TSnapshotRequest snapshot_request);
diff --git a/gensrc/thrift/Data.thrift b/gensrc/thrift/Data.thrift
index dc1190c6e425593..17007a0852671f5 100644
--- a/gensrc/thrift/Data.thrift
+++ b/gensrc/thrift/Data.thrift
@@ -20,32 +20,6 @@ namespace java org.apache.doris.thrift
 
 include "Types.thrift"
 
-// Serialized, self-contained version of a RowBatch (in be/src/runtime/row-batch.h).
-struct TRowBatch {
-  // total number of rows contained in this batch
-  1: required i32 num_rows
-
-  // row composition
-  2: required list<Types.TTupleId> row_tuples
-
-  // There are a total of num_rows * num_tuples_per_row offsets
-  // pointing into tuple_data.
-  // An offset of -1 records a NULL.
-  3: list<i32> tuple_offsets
-
-  // binary tuple data
-  // TODO: figure out how we can avoid copying the data during TRowBatch construction
-  4: string tuple_data
-
-  // Indicates whether tuple_data is snappy-compressed
-  5: bool is_compressed
-
-  // backend num, source
-  6: i32 be_number
-  // packet seq
-  7: i64 packet_seq
-}
-
 // this is a union over all possible return types
 struct TCell {
   // TODO: use <type>_val instead of camelcase
diff --git a/gensrc/thrift/Ddl.thrift b/gensrc/thrift/Ddl.thrift
deleted file mode 100644
index 9696230af909edc..000000000000000
--- a/gensrc/thrift/Ddl.thrift
+++ /dev/null
@@ -1,365 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-namespace cpp doris
-namespace java org.apache.doris.thrift
-
-include "Partitions.thrift"
-include "Types.thrift"
-include "Status.thrift"
-
-struct TDdlResult {
-  // required in V1
-  1: optional Status.TStatus status
-}
-
-enum TCommonDdlType {
-    CREATE_DATABASE
-    DROP_DATABASE
-    CREATE_TABLE
-    DROP_TABLE
-    LOAD 
-}
-
-// Parameters of CREATE DATABASE command
-struct TCreateDbParams {
-    // database name to create
-    1: required string database_name
-}
-
-// Parameters of DROP DATABASE command
-struct TDropDbParams {
-    // database name to drop
-    1: required string database_name
-}
-
-// database_name + table_name
-struct TTableName {
-    1: required string db_name
-    2: required string table_name
-}
-
-// supported aggregation type
-enum TAggType {
-    AGG_SUM
-    AGG_MIN
-    AGG_MAX
-    AGG_REPLACE
-}
-
-// column defination
-//struct TColumn {
-//    // column name
-//    1: required string column_name
-//
-//    // column type
-//    2: required Types.TColumnType column_type
-//
-//    // aggregation type, if not set, this column is a KEY column, otherwise is a value column
-//    3: optional TAggType agg_type
-//    
-//    // default value
-//    4: optional string default_value 
-//}
-
-enum THashType {
-    CRC32
-}
-
-// random partition info
-struct TRandomPartitionDesc {
-}
-
-// hash partition info
-struct THashPartitionDesc {
-    // column to compute hash value
-    1: required list<string> column_list
-
-    // hash buckets
-    2: required i32 hash_buckets
-
-    // type to compute hash value. if not set, use CRC32
-    3: optional THashType hash_type
-}
-
-// value used to represents one column value in one range value
-struct TValue {
-    1: optional string value
-
-    // if this sign is set and is true, this value is stand for MAX value
-    2: optional bool max_sign
-}
-
-// one range value
-struct TRangeValue {
-    1: required list<TValue> value_list
-}
-
-// range partition defination
-struct TRangePartitionDesc {
-    // column used to compute range
-    1: required list<string> column_list
-
-    // range value for range, if not set, all in one range
-    2: optional list<TRangeValue> range_value
-}
-
-// partition info
-struct TPartitionDesc {
-    // partition type
-    1: required Partitions.TPartitionType type
-    // hash buckets
-    2: required i32 partition_num
-
-    // hash partition information
-    3: optional THashPartitionDesc hash_partition
-
-    // range partition information
-    4: optional TRangePartitionDesc range_partition
-
-    // random partition infomation
-    5: optional TRandomPartitionDesc random_partition
-}
-
-// Parameters of CREATE TABLE command
-struct TCreateTableParams {
-    // table name to create
-    1: required TTableName table_name
-
-    // column defination.
-    // 2: required list<TColumn> columns
-
-    // engine type, if not set, use the default type.
-    3: optional string engine_name
-
-    // if set and true, no error when there is already table with same name
-    4: optional bool if_not_exists
-
-    // partition info, if not set, use the default partition type which meta define.
-    5: optional TPartitionDesc partition_desc
-
-    // used to set row format, maybe columnar or row format
-    6: optional string row_format_type
-
-    // other properties
-    7: optional map<string, string> properties
-}
-
-// Parameters of DROP TABLE command
-struct TDropTableParams {
-    // table name to drop
-    1: required TTableName table_name
-
-    // If true, no error is raised if the target db does not exist
-    2: optional bool if_exists
-}
-
-// Parameters to CREATE ROLLUP
-struct TCreateRollupParams {
-    // table name which create rollup
-    1: required TTableName table_name
-
-    // column names ROLLUP contains
-    2: required list<string> column_names
-
-    // rollup name, if not set, meta will assign a default value
-    3: optional string rollup_name
-
-    // partition info, if not set, use the base table's 
-    4: optional TPartitionDesc partition_desc
-}
-
-// Parameters to DROP ROLLUP
-struct TDropRollupParams {
-    // table name which create rollup
-    1: required TTableName table_name
-
-    // rollup name to drop
-    2: required string rollup_name
-}
-
-// Parameters for SCHEMA CHANGE
-// struct TShcemaChangeParams {
-//     // table name need to schema change
-//     1: required TTableName table_name
-// 
-//     // column definations for this table
-//     2: required list<TColumn> column_defs
-// 
-//     // rollup schema, map is 'rollup_name' -> 'list of column_name'
-//     3: required map<string, list<string>> rollup_defs
-// }
-
-// Parameters to create function
-struct TCreateFunctionParams {
-    // database name which function to create is in
-    1: required string db_name
-
-    // function name to create
-    2: required string function_name
-
-    // function argument type
-    3: required list<Types.TColumnType> argument_type
-
-    // function return type
-    4: required Types.TColumnType return_type
-
-    // function dynamic library path
-    5: required string so_file_path
-
-    // other properties
-    6: optional map<string, string> properties
-}
-
-// Parameters to drop function
-struct TDropFunctionParams {
-    // database name which function to drop is in
-    1: required string db_name
-
-    // function name to drop
-    2: required string function_name
-}
-
-// enum TSetType {
-//     SESSION
-//     GLOBAL
-// }
-// 
-// // Parameters to SET opration
-// struct TSetParams {
-//     // set type, GLOBAL\SESSION
-//     1: required TSetType type
-// 
-//     // set pairs, one name and one Expr
-//     // 2: required map<string, TExpr> set_content
-// }
-
-struct TUserSpecification {
-    1: required string user_name
-    2: optional string host_name
-}
-
-// Parameters to create user
-struct TCreateUserParams {
-    1: required TUserSpecification user_spec
-
-    // user's password
-    2: optional string password 
-}
-
-// Parameters to drop user
-struct TDropUserParams {
-    // user name to drop
-    1: required string user_spec
-}
-
-// Parameters to SET PASSWORD
-struct TSetPasswordParams {
-    1: required TUserSpecification user_spec
-
-    // password will changed to after this opration
-    3: required string password
-}
-
-enum TPrivType {
-    PRIVILEGE_READ_ONLY
-    PRIVILEGE_READ_WRITE
-}
-
-// Parameters to GRANT
-struct TGrantParams {
-    1: required TUserSpecification user_spec
-
-    // database to grant
-    3: required string db_name
-
-    // privileges to grant
-    4: required list<TPrivType> priv_types
-}
-
-// Data info 
-struct TDataSpecification {
-    // database name which table belongs to
-    1: required TTableName table_name
-
-    // all file pathes need to load
-    3: required list<string> file_path
-
-    // column names in file
-    4: optional list<string> columns
-
-    // column separator
-    5: optional string column_separator
-
-    // line separator
-    6: optional string line_separator
-
-    // if true, value will be multiply with -1
-    7: optional bool is_negative
-}
-
-struct TLabelName {
-    // database name which load_label belongs to 
-    1: required string db_name
-
-    // load label which to be canceled.
-    2: required string load_label
-}
-
-// Parameters to LOAD file
-struct TLoadParams {
-    // label belong to this load job, used when cancel load, show load
-    1: required TLabelName load_label
-
-    // data profiles used to load in this job
-    2: required list<TDataSpecification> data_profiles
-    
-    // task info
-    3: optional map<string, string> properties
-}
-
-// Parameters to CANCEL LOAD file
-struct TCancelLoadParams {
-    1: required TLabelName load_label
-}
-
-enum TPaloInternalServiceVersion {
-    V1
-}
-
-struct TMasterDdlRequest {
-    1: required TPaloInternalServiceVersion protocol_version
-    2: required TCommonDdlType ddl_type
-    3: optional TCreateDbParams create_db_params
-    4: optional TDropDbParams drop_db_params
-    // 5: optional TCreateTableParams create_table_params
-    6: optional TDropTableParams drop_table_params
-    7: optional TLoadParams load_params
-    8: optional TCancelLoadParams cancel_load_params
-    9: optional TCreateUserParams create_user_params
-    10: optional TDropUserParams drop_user_params
-    11: optional TCreateRollupParams create_rollup_params
-    12: optional TDropRollupParams drop_rollup_params
-    13: optional TCreateFunctionParams create_function_params
-    14: optional TDropFunctionParams drop_function_params
-}
-
-struct TMasterDdlResponse {
-    1: required TPaloInternalServiceVersion protocol_version
-    2: required TCommonDdlType ddl_type
-    3: optional Status.TStatus status
-}
diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift
index 8f7e3432a78bd6a..793f6f669f239ca 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -35,15 +35,6 @@ include "HeartbeatService.thrift"
 // These are supporting structs for JniFrontend.java, which serves as the glue
 // between our C++ execution environment and the Java frontend.
 
-struct TSetSessionParams {
-    1: required string user
-}
-
-struct TAuthenticateParams {
-    1: required string user
-    2: required string passwd
-}
-
 struct TColumnDesc {
   1: required string columnName
   2: required Types.TPrimitiveType columnType
@@ -64,23 +55,6 @@ struct TColumnDef {
   2: optional string comment
 }
 
-// Arguments to DescribeTable, which returns a list of column descriptors for a
-// given table
-struct TDescribeTableParams {
-  1: optional string db
-  2: required string table_name
-  3: optional string user   // deprecated
-  4: optional string user_ip    // deprecated
-  5: optional Types.TUserIdentity current_user_ident // to replace the user and user ip
-  6: optional bool show_hidden_columns = false
-  7: optional string catalog
-}
-
-// Results of a call to describeTable()
-struct TDescribeTableResult {
-  1: required list<TColumnDef> columns
-}
-
 // Arguments to DescribeTables, which returns a list of column descriptors for
 // given tables
 struct TDescribeTablesParams {
@@ -110,197 +84,6 @@ struct TShowVariableResult {
     1: required list<list<string>> variables
 }
 
-// Valid table file formats
-enum TFileFormat {
-  PARQUETFILE,
-  RCFILE,
-  SEQUENCEFILE,
-  TEXTFILE,
-}
-
-// set type
-enum TSetType {
-  OPT_DEFAULT,
-  OPT_GLOBAL,
-  OPT_SESSION,
-}
-
-// The row format specifies how to interpret the fields (columns) and lines (rows) in a
-// data file when creating a new table.
-struct TTableRowFormat {
-  // Optional terminator string used to delimit fields (columns) in the table
-  1: optional string field_terminator
-
-  // Optional terminator string used to delimit lines (rows) in a table
-  2: optional string line_terminator
-
-  // Optional string used to specify a special escape character sequence
-  3: optional string escaped_by
-}
-
-
-// Represents a single item in a partition spec (column name + value)
-struct TPartitionKeyValue {
-  // Partition column name
-  1: required string name,
-
-  // Partition value
-  2: required string value
-}
-
-// Per-client session state
-struct TSessionState {
-  // The default database, changed by USE <database> queries.
-  1: required string database
-
-  // The user who this session belongs to.
-  2: required string user
-
-  // The user who this session belongs to.
-  3: required i64 connection_id
-}
-
-struct TClientRequest {
-  // select stmt to be executed
-  1: required string stmt
-
-  // query options
-  2: required PaloInternalService.TQueryOptions queryOptions
-
-  // session state
-  3: required TSessionState sessionState;
-}
-
-
-// Parameters for SHOW DATABASES commands
-struct TExplainParams {
-  // Optional pattern to match database names. If not set, all databases are returned.
-  1: required string explain
-}
-
-struct TSetVar{
-    1: required TSetType type
-    2: required string variable
-    3: required Exprs.TExpr value
-}
-// Parameters for Set commands
-struct TSetParams {
-  // Optional pattern to match database names. If not set, all databases are returned.
-  1: required list<TSetVar> set_vars
-}
-
-struct TKillParams {
-  // Optional pattern to match database names. If not set, all databases are returned.
-  1: required bool is_kill_connection
-  2: required i64 connection_id
-}
-
-struct TCommonDdlParams {
-  //1: required Ddl.TCommonDdlType ddl_type
-  //2: optional Ddl.TCreateDbParams create_db_params
-  //3: optional Ddl.TCreateTableParams create_table_params
-  //4: optional Ddl.TLoadParams load_params
-}
-
-// Parameters for the USE db command
-struct TUseDbParams {
-  1: required string db
-}
-
-struct TResultSetMetadata {
-  1: required list<TColumnDesc> columnDescs
-}
-
-// Result of call to PaloPlanService/JniFrontend.CreateQueryRequest()
-struct TQueryExecRequest {
-  // global descriptor tbl for all fragments
-  1: optional Descriptors.TDescriptorTable desc_tbl
-
-  // fragments[i] may consume the output of fragments[j > i];
-  // fragments[0] is the root fragment and also the coordinator fragment, if
-  // it is unpartitioned.
-  2: required list<Planner.TPlanFragment> fragments
-
-  // Specifies the destination fragment of the output of each fragment.
-  // parent_fragment_idx.size() == fragments.size() - 1 and
-  // fragments[i] sends its output to fragments[dest_fragment_idx[i-1]]
-  3: optional list<i32> dest_fragment_idx
-
-  // A map from scan node ids to a list of scan range locations.
-  // The node ids refer to scan nodes in fragments[].plan_tree
-  4: optional map<Types.TPlanNodeId, list<Planner.TScanRangeLocations>>
-      per_node_scan_ranges
-
-  // Metadata of the query result set (only for select)
-  5: optional TResultSetMetadata result_set_metadata
-
-  7: required PaloInternalService.TQueryGlobals query_globals
-
-  // The statement type governs when the coordinator can judge a query to be finished.
-  // DML queries are complete after Wait(), SELECTs may not be.
-  9: required Types.TStmtType stmt_type
-
-  // The statement type governs when the coordinator can judge a query to be finished.
-  // DML queries are complete after Wait(), SELECTs may not be.
-  10: optional bool is_block_query;
-}
-
-enum TDdlType {
-  USE,
-  DESCRIBE,
-  SET,
-  EXPLAIN,
-  KILL,
-  COMMON
-}
-
-struct TDdlExecRequest {
-  1: required TDdlType ddl_type
-
-  // Parameters for USE commands
-  2: optional TUseDbParams use_db_params;
-
-  // Parameters for DESCRIBE table commands
-  3: optional TDescribeTableParams describe_table_params
-
-  10: optional TExplainParams explain_params
-
-  11: optional TSetParams set_params
-  12: optional TKillParams kill_params
-  //13: optional Ddl.TMasterDdlRequest common_params
-}
-
-// Results of an EXPLAIN
-struct TExplainResult {
-    // each line in the explain plan occupies an entry in the list
-    1: required list<Data.TResultRow> results
-}
-
-// Result of call to createExecRequest()
-struct TExecRequest {
-  1: required Types.TStmtType stmt_type;
-
-  2: optional string sql_stmt;
-
-  // Globally unique id for this request. Assigned by the planner.
-  3: required Types.TUniqueId request_id
-
-  // Copied from the corresponding TClientRequest
-  4: required PaloInternalService.TQueryOptions query_options;
-
-  // TQueryExecRequest for the backend
-  // Set iff stmt_type is QUERY or DML
-  5: optional TQueryExecRequest query_exec_request
-
-  // Set iff stmt_type is DDL
-  6: optional TDdlExecRequest ddl_exec_request
-
-  // Metadata of the query result set (not set for DML)
-  7: optional TResultSetMetadata result_set_metadata
-
-  // Result of EXPLAIN. Set iff stmt_type is EXPLAIN
-  8: optional TExplainResult explain_result
-}
 
 // Arguments to getDbNames, which returns a list of dbs that match an optional
 // pattern
@@ -1718,7 +1501,6 @@ struct TFetchRunningQueriesRequest {
 service FrontendService {
     TGetDbsResult getDbNames(1: TGetDbsParams params)
     TGetTablesResult getTableNames(1: TGetTablesParams params)
-    TDescribeTableResult describeTable(1: TDescribeTableParams params)
     TDescribeTablesResult describeTables(1: TDescribeTablesParams params)
     TShowVariableResult showVariables(1: TShowVariableRequest params)
     TReportExecStatusResult reportExecStatus(1: TReportExecStatusParams params)
diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift
index 39f3b65818f2b93..c219daef5d1ad2b 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -27,7 +27,6 @@ include "Planner.thrift"
 include "DataSinks.thrift"
 include "Data.thrift"
 include "RuntimeProfile.thrift"
-include "PaloService.thrift"
 
 // constants for TQueryOptions.num_nodes
 const i32 NUM_NODES_ALL = 0
@@ -623,11 +622,6 @@ struct TCancelPlanFragmentResult {
   1: optional Status.TStatus status
 }
 
-// fold constant expr
-struct TExprMap {
-  1: required map<string, Exprs.TExpr> expr_map
-}
-
 struct TFoldConstantParams {
   1: required map<string, map<string, Exprs.TExpr>> expr_map
   2: required TQueryGlobals query_globals
@@ -650,9 +644,6 @@ struct TTransmitDataParams {
   // required in V1
   4: optional Types.TPlanNodeId dest_node_id
 
-  // required in V1
-  5: optional Data.TRowBatch row_batch
-
   // if set to true, indicates that no more row batches will be sent
   // for this dest_node_id
   6: optional bool eos
@@ -677,66 +668,6 @@ struct TTabletWithPartition {
     2: required i64 tablet_id
 }
 
-// open a tablet writer
-struct TTabletWriterOpenParams {
-    1: required Types.TUniqueId id
-    2: required i64 index_id
-    3: required i64 txn_id
-    4: required Descriptors.TOlapTableSchemaParam schema
-    5: required list<TTabletWithPartition> tablets
-
-    6: required i32 num_senders
-}
-
-struct TTabletWriterOpenResult {
-    1: required Status.TStatus status
-}
-
-// add batch to tablet writer
-struct TTabletWriterAddBatchParams {
-    1: required Types.TUniqueId id
-    2: required i64 index_id
-
-    3: required i64 packet_seq
-    4: required list<Types.TTabletId> tablet_ids
-    5: required Data.TRowBatch row_batch
-
-    6: required i32 sender_no
-}
-
-struct TTabletWriterAddBatchResult {
-    1: required Status.TStatus status
-}
-
-struct TTabletWriterCloseParams {
-    1: required Types.TUniqueId id
-    2: required i64 index_id
-
-    3: required i32 sender_no
-}
-
-struct TTabletWriterCloseResult {
-    1: required Status.TStatus status
-}
-
-//
-struct TTabletWriterCancelParams {
-    1: required Types.TUniqueId id
-    2: required i64 index_id
-
-    3: required i32 sender_no
-}
-
-struct TTabletWriterCancelResult {
-}
-
-struct TFetchDataParams {
-  1: required PaloInternalServiceVersion protocol_version
-  // required in V1
-  // query id which want to fetch data
-  2: required Types.TUniqueId fragment_instance_id
-}
-
 struct TFetchDataResult {
     // result batch
     1: required Data.TResultBatch result_batch
diff --git a/gensrc/thrift/PaloService.thrift b/gensrc/thrift/PaloService.thrift
deleted file mode 100644
index 6dd7743244e8e2b..000000000000000
--- a/gensrc/thrift/PaloService.thrift
+++ /dev/null
@@ -1,100 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-namespace cpp doris
-namespace java org.apache.doris.thrift
-
-include "Status.thrift"
-
-// PaloService accepts query execution options through beeswax.Query.configuration in
-// key:value form. For example, the list of strings could be:
-//     "num_nodes:1", "abort_on_error:false"
-// The valid keys are listed in this enum. They map to TQueryOptions.
-// Note: If you add an option or change the default, you also need to update:
-// - PaloService.DEFAULT_QUERY_OPTIONS
-// - PaloInternalService.thrift: TQueryOptions
-// - PalodClientExecutor.getBeeswaxQueryConfigurations()
-// - PaloServer::SetQueryOptions()
-// - PaloServer::TQueryOptionsToMap()
-enum TPaloQueryOptions {
-  // if true, abort execution on the first error
-  ABORT_ON_ERROR,
-  
-  // maximum # of errors to be reported; Unspecified or 0 indicates backend default
-  MAX_ERRORS,
-  
-  // if true, disable llvm codegen
-  DISABLE_CODEGEN,
-  
-  // batch size to be used by backend; Unspecified or a size of 0 indicates backend
-  // default
-  BATCH_SIZE,
-
-  // a per-machine approximate limit on the memory consumption of this query;
-  // unspecified or a limit of 0 means no limit;
-  // otherwise specified either as:
-  // a) an int (= number of bytes);
-  // b) a float followed by "M" (MB) or "G" (GB)
-  MEM_LIMIT,
-   
-  // specifies the degree of parallelism with which to execute the query;
-  // 1: single-node execution
-  // NUM_NODES_ALL: executes on all nodes that contain relevant data
-  // NUM_NODES_ALL_RACKS: executes on one node per rack that holds relevant data
-  // > 1: executes on at most that many nodes at any point in time (ie, there can be
-  //      more nodes than numNodes with plan fragments for this query, but at most
-  //      numNodes would be active at any point in time)
-  // Constants (NUM_NODES_ALL, NUM_NODES_ALL_RACKS) are defined in JavaConstants.thrift.
-  NUM_NODES,
-  
-  // maximum length of the scan range; only applicable to HDFS scan range; Unspecified or
-  // a length of 0 indicates backend default;  
-  MAX_SCAN_RANGE_LENGTH,
-  
-  // Maximum number of io buffers (per disk)
-  MAX_IO_BUFFERS,
-
-  // Number of scanner threads.
-  NUM_SCANNER_THREADS,
-
-  QUERY_TIMEOUT,
-
-  // If true, Palo will try to execute on file formats that are not fully supported yet
-  ALLOW_UNSUPPORTED_FORMATS,
-
-  // if set and > -1, specifies the default limit applied to a top-level SELECT statement
-  // with an ORDER BY but without a LIMIT clause (ie, if the SELECT statement also has
-  // a LIMIT clause, this default is ignored)
-  DEFAULT_ORDER_BY_LIMIT,
-
-  // DEBUG ONLY:
-  // If set to
-  //   "[<backend number>:]<node id>:<TExecNodePhase>:<TDebugAction>",
-  // the exec node with the given id will perform the specified action in the given
-  // phase. If the optional backend number (starting from 0) is specified, only that
-  // backend instance will perform the debug action, otherwise all backends will behave
-  // in that way.
-  // If the string doesn't have the required format or if any of its components is
-  // invalid, the option is ignored. 
-  DEBUG_ACTION,
-  
-  // If true, raise an error when the DEFAULT_ORDER_BY_LIMIT has been reached.
-  ABORT_ON_DEFAULT_LIMIT_EXCEEDED,
-
-  // If false, the backend dosn't report the success status to coordiator
-  IS_REPORT_SUCCESS,
-}
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index 1345cac66cb12f0..9e7c90908f43290 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -61,21 +61,6 @@ enum TPlanNodeType {
   GROUP_COMMIT_SCAN_NODE
 }
 
-// phases of an execution node
-enum TExecNodePhase {
-  PREPARE,
-  OPEN,
-  GETNEXT,
-  CLOSE,
-  INVALID
-}
-
-// what to do when hitting a debug point (TPaloQueryOptions.DEBUG_ACTION)
-enum TDebugAction {
-  WAIT,
-  FAIL
-}
-
 struct TKeyRange {
   1: required i64 begin_key
   2: required i64 end_key
diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift
index c40e6a0483884eb..4dbc1d060542048 100644
--- a/gensrc/thrift/Types.thrift
+++ b/gensrc/thrift/Types.thrift
@@ -243,13 +243,6 @@ enum TTaskType {
     CALCULATE_DELETE_BITMAP = 1000
 }
 
-enum TStmtType {
-  QUERY,
-  DDL,  // Data definition, e.g. CREATE TABLE (includes read-only functions e.g. SHOW)
-  DML,  // Data modification e.g. INSERT
-  EXPLAIN   // EXPLAIN
-}
-
 // level of verboseness for "explain" output
 // TODO: should this go somewhere else?
 enum TExplainLevel {

From 1320116eb8b3f6f2061b1a2a9c6bb18835229b4f Mon Sep 17 00:00:00 2001
From: 924060929 <lanhuajian@selectdb.com>
Date: Tue, 7 Jan 2025 19:49:01 +0800
Subject: [PATCH 026/140] [fix](cache) fix mtmv cache can not use test (#46537)

fix mtmv can not use sql cache, introduced by #46443
---
 .../org/apache/doris/common/cache/NereidsSqlCacheManager.java | 2 +-
 .../suites/mtmv_p0/test_mtmv_sql_cache_and_profile.groovy     | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java
index 2aca4d5f1efac75..bf82795450048a6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java
@@ -325,7 +325,7 @@ private boolean tablesOrDataChanged(Env env, SqlCacheContext sqlCacheContext) {
         // check table type and version
         for (Entry<FullTableName, TableVersion> scanTable : sqlCacheContext.getUsedTables().entrySet()) {
             TableVersion tableVersion = scanTable.getValue();
-            if (tableVersion.type != TableType.OLAP) {
+            if (tableVersion.type != TableType.OLAP && tableVersion.type != TableType.MATERIALIZED_VIEW) {
                 return true;
             }
             TableIf tableIf = findTableIf(env, scanTable.getKey());
diff --git a/regression-test/suites/mtmv_p0/test_mtmv_sql_cache_and_profile.groovy b/regression-test/suites/mtmv_p0/test_mtmv_sql_cache_and_profile.groovy
index 02191f0d4cd7d43..846c42f138f3cf5 100644
--- a/regression-test/suites/mtmv_p0/test_mtmv_sql_cache_and_profile.groovy
+++ b/regression-test/suites/mtmv_p0/test_mtmv_sql_cache_and_profile.groovy
@@ -63,12 +63,13 @@ suite("test_mtmv_sql_cache_and_profile", "mtmv") {
     waitingMTMVTaskFinished(jobName)
 
     sql """set enable_sql_cache=true;"""
+    sql "ADMIN SET FRONTEND CONFIG ('cache_last_version_interval_second' = '10')"
+    sleep(10 * 1000)
 
     long startTime = System.currentTimeMillis()
     long timeoutTimestamp = startTime + 5 * 60 * 1000
     def explain_res = ""
     while (System.currentTimeMillis() < timeoutTimestamp) {
-        sleep(5 * 1000)
         sql """select k2 from ${mvName} group by k2;"""
         try {
             explain_res = sql """explain plan select k2 from ${mvName} group by k2;"""
@@ -79,6 +80,7 @@ suite("test_mtmv_sql_cache_and_profile", "mtmv") {
         if (explain_res.toString().indexOf("LogicalSqlCache") != -1 || explain_res.toString().indexOf("PhysicalSqlCache") != -1) {
             break
         }
+        sleep(5 * 1000)
     }
     assertTrue(explain_res.toString().indexOf("LogicalSqlCache") != -1 || explain_res.toString().indexOf("PhysicalSqlCache") != -1)
 

From b088cf5f73eff640e5e7d09679033d9f7105cc38 Mon Sep 17 00:00:00 2001
From: bobhan1 <baohan@selectdb.com>
Date: Tue, 7 Jan 2025 22:21:13 +0800
Subject: [PATCH 027/140] [Fix](partial update) abort partial update on shadow
 index's tablet when the including columns miss key columns on new schema
 (#46347)

During a schema change which changes key columns, partial update on
shadow index's tablet may cause duplicate key problem. So we abort the
partial update in this situation.
---
 be/src/cloud/cloud_rowset_builder.cpp         |  4 +-
 be/src/olap/delta_writer_v2.cpp               | 24 +++---
 be/src/olap/delta_writer_v2.h                 |  6 +-
 be/src/olap/partial_update_info.cpp           | 30 ++++++--
 be/src/olap/partial_update_info.h             | 11 +--
 .../segment_v2/vertical_segment_writer.cpp    |  8 ++
 be/src/olap/rowset_builder.cpp                | 18 +++--
 be/src/olap/rowset_builder.h                  |  6 +-
 be/src/olap/schema_change.cpp                 |  1 +
 .../apache/doris/alter/SchemaChangeJobV2.java |  1 +
 .../test_add_key_partial_update.out           | 17 +++++
 .../test_add_key_partial_update.groovy        | 74 +++++++++++++++++++
 12 files changed, 162 insertions(+), 38 deletions(-)
 create mode 100644 regression-test/data/fault_injection_p0/partial_update/test_add_key_partial_update.out
 create mode 100644 regression-test/suites/fault_injection_p0/partial_update/test_add_key_partial_update.groovy

diff --git a/be/src/cloud/cloud_rowset_builder.cpp b/be/src/cloud/cloud_rowset_builder.cpp
index 2e6764b33aa79cb..9466dd1062803e0 100644
--- a/be/src/cloud/cloud_rowset_builder.cpp
+++ b/be/src/cloud/cloud_rowset_builder.cpp
@@ -51,8 +51,8 @@ Status CloudRowsetBuilder::init() {
             duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count();
 
     // build tablet schema in request level
-    _build_current_tablet_schema(_req.index_id, _req.table_schema_param.get(),
-                                 *_tablet->tablet_schema());
+    RETURN_IF_ERROR(_build_current_tablet_schema(_req.index_id, _req.table_schema_param.get(),
+                                                 *_tablet->tablet_schema()));
 
     RowsetWriterContext context;
     context.txn_id = _req.txn_id;
diff --git a/be/src/olap/delta_writer_v2.cpp b/be/src/olap/delta_writer_v2.cpp
index a6fb01544890429..f098d94a3893335 100644
--- a/be/src/olap/delta_writer_v2.cpp
+++ b/be/src/olap/delta_writer_v2.cpp
@@ -102,8 +102,8 @@ Status DeltaWriterV2::init() {
     if (_streams.size() == 0 || _streams[0]->tablet_schema(_req.index_id) == nullptr) {
         return Status::InternalError("failed to find tablet schema for {}", _req.index_id);
     }
-    _build_current_tablet_schema(_req.index_id, _req.table_schema_param.get(),
-                                 *_streams[0]->tablet_schema(_req.index_id));
+    RETURN_IF_ERROR(_build_current_tablet_schema(_req.index_id, _req.table_schema_param.get(),
+                                                 *_streams[0]->tablet_schema(_req.index_id)));
     RowsetWriterContext context;
     context.txn_id = _req.txn_id;
     context.load_id = _req.load_id;
@@ -210,9 +210,9 @@ Status DeltaWriterV2::cancel_with_status(const Status& st) {
     return Status::OK();
 }
 
-void DeltaWriterV2::_build_current_tablet_schema(int64_t index_id,
-                                                 const OlapTableSchemaParam* table_schema_param,
-                                                 const TabletSchema& ori_tablet_schema) {
+Status DeltaWriterV2::_build_current_tablet_schema(int64_t index_id,
+                                                   const OlapTableSchemaParam* table_schema_param,
+                                                   const TabletSchema& ori_tablet_schema) {
     _tablet_schema->copy_from(ori_tablet_schema);
     // find the right index id
     int i = 0;
@@ -236,12 +236,14 @@ void DeltaWriterV2::_build_current_tablet_schema(int64_t index_id,
     }
     // set partial update columns info
     _partial_update_info = std::make_shared<PartialUpdateInfo>();
-    _partial_update_info->init(*_tablet_schema, table_schema_param->unique_key_update_mode(),
-                               table_schema_param->partial_update_input_columns(),
-                               table_schema_param->is_strict_mode(),
-                               table_schema_param->timestamp_ms(),
-                               table_schema_param->nano_seconds(), table_schema_param->timezone(),
-                               table_schema_param->auto_increment_coulumn());
+    RETURN_IF_ERROR(_partial_update_info->init(
+            _req.tablet_id, _req.txn_id, *_tablet_schema,
+            table_schema_param->unique_key_update_mode(),
+            table_schema_param->partial_update_input_columns(),
+            table_schema_param->is_strict_mode(), table_schema_param->timestamp_ms(),
+            table_schema_param->nano_seconds(), table_schema_param->timezone(),
+            table_schema_param->auto_increment_coulumn()));
+    return Status::OK();
 }
 
 } // namespace doris
diff --git a/be/src/olap/delta_writer_v2.h b/be/src/olap/delta_writer_v2.h
index f9c2800a68f499d..e4506ea0d2339db 100644
--- a/be/src/olap/delta_writer_v2.h
+++ b/be/src/olap/delta_writer_v2.h
@@ -85,9 +85,9 @@ class DeltaWriterV2 {
     Status cancel_with_status(const Status& st);
 
 private:
-    void _build_current_tablet_schema(int64_t index_id,
-                                      const OlapTableSchemaParam* table_schema_param,
-                                      const TabletSchema& ori_tablet_schema);
+    Status _build_current_tablet_schema(int64_t index_id,
+                                        const OlapTableSchemaParam* table_schema_param,
+                                        const TabletSchema& ori_tablet_schema);
 
     void _update_profile(RuntimeProfile* profile);
 
diff --git a/be/src/olap/partial_update_info.cpp b/be/src/olap/partial_update_info.cpp
index c0e83eab005e7ed..1ca269a0f317b7b 100644
--- a/be/src/olap/partial_update_info.cpp
+++ b/be/src/olap/partial_update_info.cpp
@@ -20,6 +20,7 @@
 #include <gen_cpp/olap_file.pb.h>
 
 #include "common/consts.h"
+#include "common/logging.h"
 #include "olap/base_tablet.h"
 #include "olap/olap_common.h"
 #include "olap/rowset/rowset.h"
@@ -32,12 +33,13 @@
 
 namespace doris {
 
-void PartialUpdateInfo::init(const TabletSchema& tablet_schema,
-                             UniqueKeyUpdateModePB unique_key_update_mode,
-                             const std::set<string>& partial_update_cols, bool is_strict_mode,
-                             int64_t timestamp_ms, int32_t nano_seconds,
-                             const std::string& timezone, const std::string& auto_increment_column,
-                             int32_t sequence_map_col_uid, int64_t cur_max_version) {
+Status PartialUpdateInfo::init(int64_t tablet_id, int64_t txn_id, const TabletSchema& tablet_schema,
+                               UniqueKeyUpdateModePB unique_key_update_mode,
+                               const std::set<string>& partial_update_cols, bool is_strict_mode,
+                               int64_t timestamp_ms, int32_t nano_seconds,
+                               const std::string& timezone,
+                               const std::string& auto_increment_column,
+                               int32_t sequence_map_col_uid, int64_t cur_max_version) {
     partial_update_mode = unique_key_update_mode;
     partial_update_input_columns = partial_update_cols;
     max_version_in_flush_phase = cur_max_version;
@@ -48,6 +50,21 @@ void PartialUpdateInfo::init(const TabletSchema& tablet_schema,
     missing_cids.clear();
     update_cids.clear();
 
+    if (partial_update_mode == UniqueKeyUpdateModePB::UPDATE_FIXED_COLUMNS) {
+        // partial_update_cols should include all key columns
+        for (std::size_t i {0}; i < tablet_schema.num_key_columns(); i++) {
+            const auto key_col = tablet_schema.column(i);
+            if (!partial_update_cols.contains(key_col.name())) {
+                auto msg = fmt::format(
+                        "Unable to do partial update on shadow index's tablet, tablet_id={}, "
+                        "txn_id={}. Missing key column {}.",
+                        tablet_id, txn_id, key_col.name());
+                LOG_WARNING(msg);
+                return Status::Aborted<false>(msg);
+            }
+        }
+    }
+
     for (auto i = 0; i < tablet_schema.num_columns(); ++i) {
         if (partial_update_mode == UniqueKeyUpdateModePB::UPDATE_FIXED_COLUMNS) {
             auto tablet_column = tablet_schema.column(i);
@@ -75,6 +92,7 @@ void PartialUpdateInfo::init(const TabletSchema& tablet_schema,
             is_fixed_partial_update() &&
             partial_update_input_columns.contains(auto_increment_column);
     _generate_default_values_for_missing_cids(tablet_schema);
+    return Status::OK();
 }
 
 void PartialUpdateInfo::to_pb(PartialUpdateInfoPB* partial_update_info_pb) const {
diff --git a/be/src/olap/partial_update_info.h b/be/src/olap/partial_update_info.h
index 1078a6eb0dddf4c..f50937d014c57f7 100644
--- a/be/src/olap/partial_update_info.h
+++ b/be/src/olap/partial_update_info.h
@@ -42,11 +42,12 @@ struct RowsetId;
 class BitmapValue;
 
 struct PartialUpdateInfo {
-    void init(const TabletSchema& tablet_schema, UniqueKeyUpdateModePB unique_key_update_mode,
-              const std::set<std::string>& partial_update_cols, bool is_strict_mode,
-              int64_t timestamp_ms, int32_t nano_seconds, const std::string& timezone,
-              const std::string& auto_increment_column, int32_t sequence_map_col_uid = -1,
-              int64_t cur_max_version = -1);
+    Status init(int64_t tablet_id, int64_t txn_id, const TabletSchema& tablet_schema,
+                UniqueKeyUpdateModePB unique_key_update_mode,
+                const std::set<std::string>& partial_update_cols, bool is_strict_mode,
+                int64_t timestamp_ms, int32_t nano_seconds, const std::string& timezone,
+                const std::string& auto_increment_column, int32_t sequence_map_col_uid = -1,
+                int64_t cur_max_version = -1);
     void to_pb(PartialUpdateInfoPB* partial_update_info) const;
     void from_pb(PartialUpdateInfoPB* partial_update_info);
     Status handle_non_strict_mode_not_found_error(const TabletSchema& tablet_schema,
diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
index 0846b0fc1186a84..61621d920eb45f3 100644
--- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
@@ -60,6 +60,7 @@
 #include "service/point_query_executor.h"
 #include "util/coding.h"
 #include "util/crc32c.h"
+#include "util/debug_points.h"
 #include "util/faststring.h"
 #include "util/key_util.h"
 #include "vec/columns/column_nullable.h"
@@ -472,6 +473,8 @@ Status VerticalSegmentWriter::_partial_update_preconditions_check(size_t row_pos
 // 3. set columns to data convertor and then write all columns
 Status VerticalSegmentWriter::_append_block_with_partial_content(RowsInBlock& data,
                                                                  vectorized::Block& full_block) {
+    DBUG_EXECUTE_IF("_append_block_with_partial_content.block", DBUG_BLOCK);
+
     RETURN_IF_ERROR(_partial_update_preconditions_check(data.row_pos, false));
     // create full block and fill with input columns
     full_block = _tablet_schema->create_block();
@@ -1340,6 +1343,11 @@ void VerticalSegmentWriter::_encode_rowid(const uint32_t rowid, string* encoded_
 std::string VerticalSegmentWriter::_full_encode_keys(
         const std::vector<vectorized::IOlapColumnDataAccessor*>& key_columns, size_t pos) {
     assert(_key_index_size.size() == _num_sort_key_columns);
+    if (!(key_columns.size() == _num_sort_key_columns &&
+          _key_coders.size() == _num_sort_key_columns)) {
+        LOG_INFO("key_columns.size()={}, _key_coders.size()={}, _num_sort_key_columns={}, ",
+                 key_columns.size(), _key_coders.size(), _num_sort_key_columns);
+    }
     assert(key_columns.size() == _num_sort_key_columns &&
            _key_coders.size() == _num_sort_key_columns);
     return _full_encode_keys(_key_coders, key_columns, pos);
diff --git a/be/src/olap/rowset_builder.cpp b/be/src/olap/rowset_builder.cpp
index 1bbacc0b1236f63..ab4fa09f34c9616 100644
--- a/be/src/olap/rowset_builder.cpp
+++ b/be/src/olap/rowset_builder.cpp
@@ -221,8 +221,8 @@ Status RowsetBuilder::init() {
         };
     })
     // build tablet schema in request level
-    _build_current_tablet_schema(_req.index_id, _req.table_schema_param.get(),
-                                 *_tablet->tablet_schema());
+    RETURN_IF_ERROR(_build_current_tablet_schema(_req.index_id, _req.table_schema_param.get(),
+                                                 *_tablet->tablet_schema()));
     RowsetWriterContext context;
     context.txn_id = _req.txn_id;
     context.load_id = _req.load_id;
@@ -396,9 +396,9 @@ Status BaseRowsetBuilder::cancel() {
     return Status::OK();
 }
 
-void BaseRowsetBuilder::_build_current_tablet_schema(int64_t index_id,
-                                                     const OlapTableSchemaParam* table_schema_param,
-                                                     const TabletSchema& ori_tablet_schema) {
+Status BaseRowsetBuilder::_build_current_tablet_schema(
+        int64_t index_id, const OlapTableSchemaParam* table_schema_param,
+        const TabletSchema& ori_tablet_schema) {
     // find the right index id
     int i = 0;
     auto indexes = table_schema_param->indexes();
@@ -438,13 +438,15 @@ void BaseRowsetBuilder::_build_current_tablet_schema(int64_t index_id,
     }
     // set partial update columns info
     _partial_update_info = std::make_shared<PartialUpdateInfo>();
-    _partial_update_info->init(
-            *_tablet_schema, table_schema_param->unique_key_update_mode(),
+    RETURN_IF_ERROR(_partial_update_info->init(
+            tablet()->tablet_id(), _req.txn_id, *_tablet_schema,
+            table_schema_param->unique_key_update_mode(),
             table_schema_param->partial_update_input_columns(),
             table_schema_param->is_strict_mode(), table_schema_param->timestamp_ms(),
             table_schema_param->nano_seconds(), table_schema_param->timezone(),
             table_schema_param->auto_increment_coulumn(),
-            table_schema_param->sequence_map_col_uid(), _max_version_in_flush_phase);
+            table_schema_param->sequence_map_col_uid(), _max_version_in_flush_phase));
+    return Status::OK();
 }
 
 } // namespace doris
diff --git a/be/src/olap/rowset_builder.h b/be/src/olap/rowset_builder.h
index fb2294d1770cc42..d87e2a9efa4a8f2 100644
--- a/be/src/olap/rowset_builder.h
+++ b/be/src/olap/rowset_builder.h
@@ -84,9 +84,9 @@ class BaseRowsetBuilder {
     Status init_mow_context(std::shared_ptr<MowContext>& mow_context);
 
 protected:
-    void _build_current_tablet_schema(int64_t index_id,
-                                      const OlapTableSchemaParam* table_schema_param,
-                                      const TabletSchema& ori_tablet_schema);
+    Status _build_current_tablet_schema(int64_t index_id,
+                                        const OlapTableSchemaParam* table_schema_param,
+                                        const TabletSchema& ori_tablet_schema);
 
     virtual void _init_profile(RuntimeProfile* profile);
 
diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index 063e5e9af5ddc4c..4a737a1316e0cd8 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -805,6 +805,7 @@ Status SchemaChangeJob::process_alter_tablet(const TAlterTabletReqV2& request) {
 
     Status res = _do_process_alter_tablet(request);
     LOG(INFO) << "finished alter tablet process, res=" << res;
+    DBUG_EXECUTE_IF("SchemaChangeJob::process_alter_tablet.leave.sleep", { sleep(5); });
     return res;
 }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
index 88fa3286c886bfa..bea703a29d75fa5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
@@ -601,6 +601,7 @@ protected void runRunningJob() throws AlterCancelException {
         Env.getCurrentEnv().getGroupCommitManager().blockTable(tableId);
         Env.getCurrentEnv().getGroupCommitManager().waitWalFinished(tableId);
         Env.getCurrentEnv().getGroupCommitManager().unblockTable(tableId);
+
         /*
          * all tasks are finished. check the integrity.
          * we just check whether all new replicas are healthy.
diff --git a/regression-test/data/fault_injection_p0/partial_update/test_add_key_partial_update.out b/regression-test/data/fault_injection_p0/partial_update/test_add_key_partial_update.out
new file mode 100644
index 000000000000000..87d44dac59e4f44
--- /dev/null
+++ b/regression-test/data/fault_injection_p0/partial_update/test_add_key_partial_update.out
@@ -0,0 +1,17 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+1	1	1
+2	2	2
+3	3	3
+4	4	4
+5	5	5
+6	6	6
+
+-- !sql --
+1	\N	1	1	2	0
+2	\N	2	2	2	0
+3	\N	3	3	2	0
+4	\N	4	4	5	0
+5	\N	5	5	5	0
+6	\N	6	6	5	0
+
diff --git a/regression-test/suites/fault_injection_p0/partial_update/test_add_key_partial_update.groovy b/regression-test/suites/fault_injection_p0/partial_update/test_add_key_partial_update.groovy
new file mode 100644
index 000000000000000..61ba9d60ea83712
--- /dev/null
+++ b/regression-test/suites/fault_injection_p0/partial_update/test_add_key_partial_update.groovy
@@ -0,0 +1,74 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Assert
+import java.util.concurrent.TimeUnit
+import org.awaitility.Awaitility
+
+suite("test_add_key_partial_update", "nonConcurrent") {
+
+    def table1 = "test_add_key_partial_update"
+    sql "DROP TABLE IF EXISTS ${table1} FORCE;"
+    sql """ CREATE TABLE IF NOT EXISTS ${table1} (
+            `k1` int NOT NULL,
+            `c1` int,
+            `c2` int,
+        )UNIQUE KEY(k1)
+        DISTRIBUTED BY HASH(k1) BUCKETS 1
+        PROPERTIES (
+            "enable_mow_light_delete" = "false",
+            "disable_auto_compaction" = "true",
+            "replication_num" = "1"); """
+
+    sql "insert into ${table1} values(1,1,1),(2,2,2),(3,3,3);"
+    sql "insert into ${table1} values(4,4,4),(5,5,5),(6,6,6);"
+    sql "insert into ${table1} values(4,4,4),(5,5,5),(6,6,6);"
+    sql "insert into ${table1} values(4,4,4),(5,5,5),(6,6,6);"
+    sql "sync;"
+    order_qt_sql "select * from ${table1};"
+
+    try {
+        GetDebugPoint().clearDebugPointsForAllFEs()
+        GetDebugPoint().clearDebugPointsForAllBEs()
+
+        // block the schema change process before it change the shadow index to base index
+        GetDebugPoint().enableDebugPointForAllBEs("SchemaChangeJob::process_alter_tablet.leave.sleep")
+
+        sql "alter table ${table1} ADD COLUMN k2 int key;"
+
+        Thread.sleep(1000)
+        test {
+            sql "delete from ${table1} where k1<=3;"
+            exception "Unable to do partial update on shadow index's tablet"
+        }
+
+        waitForSchemaChangeDone {
+            sql """ SHOW ALTER TABLE COLUMN WHERE TableName='${table1}' ORDER BY createtime DESC LIMIT 1 """
+            time 1000
+        }
+
+        sql "set skip_delete_sign=true;"
+        sql "sync;"
+        qt_sql "select k1,k2,c1,c2,__DORIS_VERSION_COL__,__DORIS_DELETE_SIGN__ from ${table1} order by k1,k2,__DORIS_VERSION_COL__;"
+    } catch(Exception e) {
+        logger.info(e.getMessage())
+        throw e
+    } finally {
+        GetDebugPoint().clearDebugPointsForAllFEs()
+        GetDebugPoint().clearDebugPointsForAllBEs()
+    }
+}

From ed07dc8596a70ab7e8aa44f0f222218b3e225663 Mon Sep 17 00:00:00 2001
From: hui lai <laihui@selectdb.com>
Date: Tue, 7 Jan 2025 22:31:52 +0800
Subject: [PATCH 028/140] [fix](cloud) fix routine load loss data when fe
 master node restart (#46149)

In cloud mode, routine load loss data when fe master node restart.

When updating progress, in order to avoid small values covering large
values, we introduced pr https://github.com/apache/doris/pull/39313, Due
to the pr that the routine load replays progress metadata by first
obtaining the set default offset and then pulling metadata from meta
service to update the local value, if the metadata pulled from meta
service is not larger than the set default offset, the correct value
cannot be assigned to memory.

To solve this problem, pulling metadata from meta service when restart,
determine whether to obtain default offset from Kafka based on the
pulled value.
---
 .../apache/doris/load/routineload/KafkaRoutineLoadJob.java   | 5 +++++
 .../apache/doris/load/routineload/RoutineLoadScheduler.java  | 5 -----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaRoutineLoadJob.java
index 8cb0898eda83d0a..0376cd3f366c3c1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaRoutineLoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaRoutineLoadJob.java
@@ -366,6 +366,11 @@ protected RoutineLoadTaskInfo unprotectRenewTask(RoutineLoadTaskInfo routineLoad
 
     @Override
     protected void unprotectUpdateProgress() throws UserException {
+        // For cloud mode, should update cloud progress from meta service,
+        // then update progress with default offset from Kafka if necessary.
+        if (Config.isCloudMode()) {
+            updateCloudProgress();
+        }
         updateNewPartitionProgress();
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadScheduler.java
index 51029c3d18b194e..023cd239e0995bc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadScheduler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadScheduler.java
@@ -18,7 +18,6 @@
 package org.apache.doris.load.routineload;
 
 import org.apache.doris.catalog.Env;
-import org.apache.doris.common.Config;
 import org.apache.doris.common.FeConstants;
 import org.apache.doris.common.LoadException;
 import org.apache.doris.common.MetaNotFoundException;
@@ -79,10 +78,6 @@ private void process() throws UserException {
             RoutineLoadJob.JobState errorJobState = null;
             UserException userException = null;
             try {
-                if (Config.isCloudMode()) {
-                    routineLoadJob.updateCloudProgress();
-                }
-
                 routineLoadJob.prepare();
                 // judge nums of tasks more than max concurrent tasks of cluster
                 int desiredConcurrentTaskNum = routineLoadJob.calculateCurrentConcurrentTaskNum();

From d7b28f584da4f225eed2c317a1a95cd5c549a40f Mon Sep 17 00:00:00 2001
From: hui lai <laihui@selectdb.com>
Date: Tue, 7 Jan 2025 22:35:38 +0800
Subject: [PATCH 029/140] [improve](routine load) improve routine load
 observability (#46238)

1. **reset other msg in the a stream window**
The routine load job is a continuously scheduled job, and as the job
runs, previous errors do not need to be constantly displayed.

2. **show error info when  transaction of sub task failed**
If a subtask fails, it will continuously retry, and there may be some
errors that prevent the job from scheduling and consuming data properly,
such as continuous too many segments error(code: -235). At this time, it
is necessary to display it in a timely manner to make the user aware.

3. **set pause reason to other msg when reschedule job**
For jobs that are unexpectedly paused, the job manager has an auto
resume mechanism. However, for some scenarios, such as not being able to
connect to Kafka and being auto resumed after pause to retry, it may
cause users to not see the problem for a long time. Unexpectedly paused
jobs always have issues, even if auto resume occurs, the reason for the
error needs to be displayed.
---
 .../load/routineload/RoutineLoadJob.java      |  15 +-
 .../routine_load/data/test_error_info.csv     |   1 +
 .../test_routine_load_error_info.groovy       | 290 ++++++++++++++++++
 3 files changed, 303 insertions(+), 3 deletions(-)
 create mode 100644 regression-test/suites/load_p0/routine_load/data/test_error_info.csv
 create mode 100644 regression-test/suites/load_p0/routine_load/test_routine_load_error_info.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
index 048c999b30eaaf7..fdefbb65afc3967 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
@@ -503,7 +503,12 @@ public long getDbId() {
     }
 
     public void setOtherMsg(String otherMsg) {
-        this.otherMsg = TimeUtils.getCurrentFormatTime() + ":" + Strings.nullToEmpty(otherMsg);
+        writeLock();
+        try {
+            this.otherMsg = TimeUtils.getCurrentFormatTime() + ":" + Strings.nullToEmpty(otherMsg);
+        } finally {
+            writeUnlock();
+        }
     }
 
     public String getDbFullName() throws MetaNotFoundException {
@@ -920,9 +925,10 @@ private void updateNumOfData(long numOfTotalRows, long numOfErrorRows, long unse
                                 + "when current total rows is more than base or the filter ratio is more than the max")
                         .build());
             }
-            // reset currentTotalNum and currentErrorNum
+            // reset currentTotalNum, currentErrorNum and otherMsg
             this.jobStatistic.currentErrorRows = 0;
             this.jobStatistic.currentTotalRows = 0;
+            this.otherMsg = "";
         } else if (this.jobStatistic.currentErrorRows > maxErrorNum
                 || (this.jobStatistic.currentTotalRows > 0
                     && ((double) this.jobStatistic.currentErrorRows
@@ -941,9 +947,10 @@ private void updateNumOfData(long numOfTotalRows, long numOfErrorRows, long unse
                         "current error rows is more than max_error_number "
                             + "or the max_filter_ratio is more than the value set"), isReplay);
             }
-            // reset currentTotalNum and currentErrorNum
+            // reset currentTotalNum, currentErrorNum and otherMsg
             this.jobStatistic.currentErrorRows = 0;
             this.jobStatistic.currentTotalRows = 0;
+            this.otherMsg = "";
         }
     }
 
@@ -1202,6 +1209,7 @@ public void afterAborted(TransactionState txnState, boolean txnOperated, String
         long taskBeId = -1L;
         try {
             this.jobStatistic.runningTxnIds.remove(txnState.getTransactionId());
+            setOtherMsg(txnStatusChangeReasonString);
             if (txnOperated) {
                 // step0: find task in job
                 Optional<RoutineLoadTaskInfo> routineLoadTaskInfoOptional = routineLoadTaskInfoList.stream().filter(
@@ -1521,6 +1529,7 @@ public void update() throws UserException {
                 LOG.info(new LogBuilder(LogKey.ROUTINE_LOAD_JOB, id)
                         .add("msg", "Job need to be rescheduled")
                         .build());
+                this.otherMsg = pauseReason == null ? "" : pauseReason.getMsg();
                 unprotectUpdateProgress();
                 unprotectUpdateState(JobState.NEED_SCHEDULE, null, false);
             }
diff --git a/regression-test/suites/load_p0/routine_load/data/test_error_info.csv b/regression-test/suites/load_p0/routine_load/data/test_error_info.csv
new file mode 100644
index 000000000000000..bc857cabcfdb5c6
--- /dev/null
+++ b/regression-test/suites/load_p0/routine_load/data/test_error_info.csv
@@ -0,0 +1 @@
+57|2023-08-19|TRUE|2|-25462|-74112029|6458082754318544493|-7910671781690629051|-15205.859375|-306870797.484914|759730669.0|-628556336.0|2023-07-10 18:39:10|2023-02-12|2023-01-27 07:26:06|y||Xi9nDVrLv8m6AwEpUxmtzFAuK48sQ|{"name": "John", "age": 25, "city": "New York"}
\ No newline at end of file
diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_error_info.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_error_info.groovy
new file mode 100644
index 000000000000000..f05b8af79ee53af
--- /dev/null
+++ b/regression-test/suites/load_p0/routine_load/test_routine_load_error_info.groovy
@@ -0,0 +1,290 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.kafka.clients.admin.AdminClient
+import org.apache.kafka.clients.producer.KafkaProducer
+import org.apache.kafka.clients.producer.ProducerRecord
+import org.apache.kafka.clients.producer.ProducerConfig
+
+suite("test_routine_load_error_info","nonConcurrent") {
+    def kafkaCsvTpoics = [
+                  "test_error_info",
+                ]
+
+    String enabled = context.config.otherConfigs.get("enableKafkaTest")
+    String kafka_port = context.config.otherConfigs.get("kafka_port")
+    String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+    def kafka_broker = "${externalEnvIp}:${kafka_port}"
+
+    // send data to kafka 
+    if (enabled != null && enabled.equalsIgnoreCase("true")) {
+        def props = new Properties()
+        props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString())
+        props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer")
+        props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer")
+        def producer = new KafkaProducer<>(props)
+        for (String kafkaCsvTopic in kafkaCsvTpoics) {
+            def txt = new File("""${context.file.parent}/data/${kafkaCsvTopic}.csv""").text
+            def lines = txt.readLines()
+            lines.each { line ->
+                logger.info("=====${line}========")
+                def record = new ProducerRecord<>(kafkaCsvTopic, null, line)
+                producer.send(record)
+            }
+        }
+    }
+
+    // case 1: task failed
+    if (enabled != null && enabled.equalsIgnoreCase("true")) {
+        // create table
+        def jobName = "test_error_info"
+        def tableName = "test_routine_error_info"
+        try {
+            sql """
+            CREATE TABLE IF NOT EXISTS ${tableName}
+            (
+                k00 INT             NOT NULL,
+                k01 DATE            NOT NULL,
+                k02 BOOLEAN         NULL,
+                k03 TINYINT         NULL,
+                k04 SMALLINT        NULL,
+                k05 INT             NULL,
+                k06 BIGINT          NULL,
+                k07 LARGEINT        NULL,
+                k08 FLOAT           NULL,
+                k09 DOUBLE          NULL,
+                k10 DECIMAL(9,1)    NULL,
+                k11 DECIMALV3(9,1)  NULL,
+                k12 DATETIME        NULL,
+                k13 DATEV2          NULL,
+                k14 DATETIMEV2      NULL,
+                k15 CHAR            NULL,
+                k16 VARCHAR         NULL,
+                k17 STRING          NULL,
+                k18 JSON            NULL,
+                kd01 BOOLEAN         NOT NULL DEFAULT "TRUE",
+                kd02 TINYINT         NOT NULL DEFAULT "1",
+                kd03 SMALLINT        NOT NULL DEFAULT "2",
+                kd04 INT             NOT NULL DEFAULT "3",
+                kd05 BIGINT          NOT NULL DEFAULT "4",
+                kd06 LARGEINT        NOT NULL DEFAULT "5",
+                kd07 FLOAT           NOT NULL DEFAULT "6.0",
+                kd08 DOUBLE          NOT NULL DEFAULT "7.0",
+                kd09 DECIMAL         NOT NULL DEFAULT "888888888",
+                kd10 DECIMALV3       NOT NULL DEFAULT "999999999",
+                kd11 DATE            NOT NULL DEFAULT "2023-08-24",
+                kd12 DATETIME        NOT NULL DEFAULT "2023-08-24 12:00:00",
+                kd13 DATEV2          NOT NULL DEFAULT "2023-08-24",
+                kd14 DATETIMEV2      NOT NULL DEFAULT "2023-08-24 12:00:00",
+                kd15 CHAR(255)       NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+                kd16 VARCHAR(300)    NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+                kd17 STRING          NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+                kd18 JSON            NULL,
+                
+                INDEX idx_inverted_k104 (`k05`) USING INVERTED,
+                INDEX idx_inverted_k110 (`k11`) USING INVERTED,
+                INDEX idx_inverted_k113 (`k13`) USING INVERTED,
+                INDEX idx_inverted_k114 (`k14`) USING INVERTED,
+                INDEX idx_inverted_k117 (`k17`) USING INVERTED PROPERTIES("parser" = "english"),
+                INDEX idx_ngrambf_k115 (`k15`) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256"),
+                INDEX idx_ngrambf_k116 (`k16`) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256"),
+                INDEX idx_ngrambf_k117 (`k17`) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256"),
+
+                INDEX idx_bitmap_k104 (`k02`) USING BITMAP,
+                INDEX idx_bitmap_k110 (`kd01`) USING BITMAP
+                
+            )
+            DUPLICATE KEY(k00)
+            PARTITION BY RANGE(k01)
+            (
+                PARTITION p1 VALUES [('2023-08-01'), ('2023-08-11')),
+                PARTITION p2 VALUES [('2023-08-11'), ('2023-08-21')),
+                PARTITION p3 VALUES [('2023-08-21'), ('2023-09-01'))
+            )
+            DISTRIBUTED BY HASH(k00) BUCKETS 32
+            PROPERTIES (
+                "bloom_filter_columns"="k05",
+                "replication_num" = "1"
+            );
+            """
+            sql "sync"
+
+            // create job
+            GetDebugPoint().enableDebugPointForAllBEs("BetaRowsetWriter._check_segment_number_limit_too_many_segments")
+            sql """
+                CREATE ROUTINE LOAD ${jobName} on ${tableName}
+                COLUMNS(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18),
+                COLUMNS TERMINATED BY "|"
+                PROPERTIES
+                (
+                    "max_batch_interval" = "5",
+                    "max_batch_rows" = "300000",
+                    "max_batch_size" = "209715200"
+                )
+                FROM KAFKA
+                (
+                    "kafka_broker_list" = "${externalEnvIp}:${kafka_port}",
+                    "kafka_topic" = "${kafkaCsvTpoics[0]}",
+                    "property.kafka_default_offsets" = "OFFSET_BEGINNING"
+                );
+            """
+            sql "sync"
+
+            // check error info
+            def count = 0
+            while (true) {
+                def res = sql "show routine load for ${jobName}"
+                log.info("show routine load: ${res[0].toString()}".toString())
+                log.info("other msg: ${res[0][19].toString()}".toString())
+                if (res[0][19].toString() != "") {
+                    assertTrue(res[0][19].toString().contains("too many segments in rowset"))
+                    break;
+                }
+                count++
+                if (count > 60) {
+                    assertEquals(1, 2)
+                    break;
+                } else {
+                    sleep(1000)
+                    continue;
+                }
+            }
+        } finally {
+            GetDebugPoint().disableDebugPointForAllBEs("BetaRowsetWriter._check_segment_number_limit_too_many_segments")
+            sql "stop routine load for ${jobName}"
+            sql "DROP TABLE IF EXISTS ${tableName}"
+        }
+    }
+
+    // case 2: reschedule job
+    if (enabled != null && enabled.equalsIgnoreCase("true")) {
+        def jobName = "test_error_info"
+        def tableName = "test_routine_error_info"
+        try {
+            sql """
+            CREATE TABLE IF NOT EXISTS ${tableName}
+            (
+                k00 INT             NOT NULL,
+                k01 DATE            NOT NULL,
+                k02 BOOLEAN         NULL,
+                k03 TINYINT         NULL,
+                k04 SMALLINT        NULL,
+                k05 INT             NULL,
+                k06 BIGINT          NULL,
+                k07 LARGEINT        NULL,
+                k08 FLOAT           NULL,
+                k09 DOUBLE          NULL,
+                k10 DECIMAL(9,1)    NULL,
+                k11 DECIMALV3(9,1)  NULL,
+                k12 DATETIME        NULL,
+                k13 DATEV2          NULL,
+                k14 DATETIMEV2      NULL,
+                k15 CHAR            NULL,
+                k16 VARCHAR         NULL,
+                k17 STRING          NULL,
+                k18 JSON            NULL,
+                kd01 BOOLEAN         NOT NULL DEFAULT "TRUE",
+                kd02 TINYINT         NOT NULL DEFAULT "1",
+                kd03 SMALLINT        NOT NULL DEFAULT "2",
+                kd04 INT             NOT NULL DEFAULT "3",
+                kd05 BIGINT          NOT NULL DEFAULT "4",
+                kd06 LARGEINT        NOT NULL DEFAULT "5",
+                kd07 FLOAT           NOT NULL DEFAULT "6.0",
+                kd08 DOUBLE          NOT NULL DEFAULT "7.0",
+                kd09 DECIMAL         NOT NULL DEFAULT "888888888",
+                kd10 DECIMALV3       NOT NULL DEFAULT "999999999",
+                kd11 DATE            NOT NULL DEFAULT "2023-08-24",
+                kd12 DATETIME        NOT NULL DEFAULT "2023-08-24 12:00:00",
+                kd13 DATEV2          NOT NULL DEFAULT "2023-08-24",
+                kd14 DATETIMEV2      NOT NULL DEFAULT "2023-08-24 12:00:00",
+                kd15 CHAR(255)       NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+                kd16 VARCHAR(300)    NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+                kd17 STRING          NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+                kd18 JSON            NULL,
+                
+                INDEX idx_inverted_k104 (`k05`) USING INVERTED,
+                INDEX idx_inverted_k110 (`k11`) USING INVERTED,
+                INDEX idx_inverted_k113 (`k13`) USING INVERTED,
+                INDEX idx_inverted_k114 (`k14`) USING INVERTED,
+                INDEX idx_inverted_k117 (`k17`) USING INVERTED PROPERTIES("parser" = "english"),
+                INDEX idx_ngrambf_k115 (`k15`) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256"),
+                INDEX idx_ngrambf_k116 (`k16`) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256"),
+                INDEX idx_ngrambf_k117 (`k17`) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256"),
+
+                INDEX idx_bitmap_k104 (`k02`) USING BITMAP,
+                INDEX idx_bitmap_k110 (`kd01`) USING BITMAP
+                
+            )
+            DUPLICATE KEY(k00)
+            PARTITION BY RANGE(k01)
+            (
+                PARTITION p1 VALUES [('2023-08-01'), ('2023-08-11')),
+                PARTITION p2 VALUES [('2023-08-11'), ('2023-08-21')),
+                PARTITION p3 VALUES [('2023-08-21'), ('2023-09-01'))
+            )
+            DISTRIBUTED BY HASH(k00) BUCKETS 32
+            PROPERTIES (
+                "bloom_filter_columns"="k05",
+                "replication_num" = "1"
+            );
+            """
+            sql "sync"
+
+            // create job
+            sql """
+                CREATE ROUTINE LOAD ${jobName} on ${tableName}
+                COLUMNS(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18),
+                COLUMNS TERMINATED BY "|"
+                PROPERTIES
+                (
+                    "max_batch_interval" = "5",
+                    "max_batch_rows" = "300000",
+                    "max_batch_size" = "209715200"
+                )
+                FROM KAFKA
+                (
+                    "kafka_broker_list" = "${externalEnvIp}:${kafka_port}",
+                    "kafka_topic" = "invalid_job",
+                    "property.kafka_default_offsets" = "OFFSET_BEGINNING"
+                );
+            """
+
+            // check error info
+            def count = 0
+            while (true) {
+                def res = sql "show routine load for ${jobName}"
+                log.info("show routine load: ${res[0].toString()}".toString())
+                log.info("other msg: ${res[0][19].toString()}".toString())
+                if (res[0][19].toString() != "" && res[0][8].toString() == "NEED_SCHEDULE") {
+                    assertTrue(res[0][19].toString().contains("may be Kafka properties set in job is error or no partition in this topic that should check Kafka"))
+                    break;
+                }
+                count++
+                if (count > 60) {
+                    assertEquals(1, 2)
+                    break;
+                } else {
+                    sleep(1000)
+                    continue;
+                }
+            }
+        } finally {
+            sql "stop routine load for ${jobName}"
+            sql "DROP TABLE IF EXISTS ${tableName}"
+        }
+    }
+}
\ No newline at end of file

From 60d57d3ab6fb6b2d1baae00280770905379b2dc8 Mon Sep 17 00:00:00 2001
From: wuwenchi <wuwenchi@selectdb.com>
Date: Wed, 8 Jan 2025 00:07:59 +0800
Subject: [PATCH 030/140] [opt](iceberg docker)Use PostgreSQL as the backend
 for the Iceberg REST server.  (#46289)

### What problem does this PR solve?

Problem Summary:
Use PostgreSQL as the backend for the Iceberg REST server. SQLite does
not support concurrent writes. When multiple engines attempt to operate
on the same database simultaneously, a database lock error will occur.
---
 .../docker-compose/iceberg/entrypoint.sh.tpl  | 37 ++++++++++---------
 .../docker-compose/iceberg/iceberg.yaml.tpl   | 19 ++++++++++
 2 files changed, 38 insertions(+), 18 deletions(-)

diff --git a/docker/thirdparties/docker-compose/iceberg/entrypoint.sh.tpl b/docker/thirdparties/docker-compose/iceberg/entrypoint.sh.tpl
index a4b27bdd6c0eec0..45d9bbf35923f5d 100644
--- a/docker/thirdparties/docker-compose/iceberg/entrypoint.sh.tpl
+++ b/docker/thirdparties/docker-compose/iceberg/entrypoint.sh.tpl
@@ -23,24 +23,25 @@ start-worker.sh spark://doris--spark-iceberg:7077
 start-history-server.sh
 start-thriftserver.sh --driver-java-options "-Dderby.system.home=/tmp/derby"
 
-
-
-ls /mnt/scripts/create_preinstalled_scripts/iceberg/*.sql | xargs -n 1 -I {} bash -c '
-    START_TIME=$(date +%s)
-    spark-sql --master spark://doris--spark-iceberg:7077 --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions -f {} 
-    END_TIME=$(date +%s)
-    EXECUTION_TIME=$((END_TIME - START_TIME))
-    echo "Script: {} executed in $EXECUTION_TIME seconds"
-'
-
-ls /mnt/scripts/create_preinstalled_scripts/paimon/*.sql | xargs -n 1 -I {} bash -c '
-    START_TIME=$(date +%s)
-    spark-sql --master  spark://doris--spark-iceberg:7077 --conf spark.sql.extensions=org.apache.paimon.spark.extensions.PaimonSparkSessionExtensions -f {} 
-    END_TIME=$(date +%s)
-    EXECUTION_TIME=$((END_TIME - START_TIME))
-    echo "Script: {} executed in $EXECUTION_TIME seconds"
-'
-
+# The creation of a Spark SQL client is time-consuming,
+# and reopening a new client for each SQL file execution leads to significant overhead.
+# To reduce the time spent on creating clients,
+# we group these files together and execute them using a single client.
+# This approach can reduce the time from 150s to 40s.
+
+START_TIME1=$(date +%s)
+find /mnt/scripts/create_preinstalled_scripts/iceberg -name '*.sql' | sed 's|^|source |' | sed 's|$|;|'> iceberg_total.sql
+spark-sql --master spark://doris--spark-iceberg:7077 --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions -f iceberg_total.sql 
+END_TIME1=$(date +%s)
+EXECUTION_TIME1=$((END_TIME1 - START_TIME1))
+echo "Script iceberg total: {} executed in $EXECUTION_TIME1 seconds"
+
+START_TIME2=$(date +%s)
+find /mnt/scripts/create_preinstalled_scripts/paimon -name '*.sql' | sed 's|^|source |' | sed 's|$|;|'> paimon_total.sql
+spark-sql --master  spark://doris--spark-iceberg:7077 --conf spark.sql.extensions=org.apache.paimon.spark.extensions.PaimonSparkSessionExtensions -f paimon_total.sql
+END_TIME2=$(date +%s)
+EXECUTION_TIME2=$((END_TIME2 - START_TIME2))
+echo "Script paimon total: {} executed in $EXECUTION_TIME2 seconds"
 
 touch /mnt/SUCCESS;
 
diff --git a/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl b/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
index 38491f645a9a3a4..fa4f8d1cca4a39f 100644
--- a/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
+++ b/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
@@ -18,6 +18,7 @@
 version: "3"
 
 services:
+
   spark-iceberg:
     image: tabulario/spark-iceberg
     container_name: doris--spark-iceberg
@@ -47,6 +48,19 @@ services:
       interval: 5s
       timeout: 120s
       retries: 120
+
+  postgres:
+    image: postgis/postgis:14-3.3
+    container_name: doris--postgres
+    environment:
+      POSTGRES_PASSWORD: 123456
+      POSTGRES_USER: root
+      POSTGRES_DB: iceberg
+    volumes:
+      - ./data/input/pgdata:/var/lib/postgresql/data
+    networks:
+      - doris--iceberg
+
   rest:
     image: tabulario/iceberg-rest
     container_name: doris--iceberg-rest
@@ -54,6 +68,8 @@ services:
       - ${REST_CATALOG_PORT}:8181
     volumes:
       - ./data:/mnt/data
+    depends_on:
+      - postgres
     environment:
       - AWS_ACCESS_KEY_ID=admin
       - AWS_SECRET_ACCESS_KEY=password
@@ -61,6 +77,9 @@ services:
       - CATALOG_WAREHOUSE=s3a://warehouse/wh/
       - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO
       - CATALOG_S3_ENDPOINT=http://minio:9000
+      - CATALOG_URI=jdbc:postgresql://postgres:5432/iceberg
+      - CATALOG_JDBC_USER=root
+      - CATALOG_JDBC_PASSWORD=123456
     networks:
       - doris--iceberg
     entrypoint: /bin/bash /mnt/data/input/script/rest_init.sh

From 3da66157b780df6e12f902ab754d37ee99850f7b Mon Sep 17 00:00:00 2001
From: abmdocrt <lianyukang@selectdb.com>
Date: Wed, 8 Jan 2025 01:31:37 +0800
Subject: [PATCH 031/140] [Fix](recycler) Fix protobuf out-of-bounds problem in
 recycler (#46484)

---
 cloud/src/meta-service/meta_service_job.cpp | 8 ++++++++
 cloud/src/recycler/recycler.cpp             | 6 +++---
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/cloud/src/meta-service/meta_service_job.cpp b/cloud/src/meta-service/meta_service_job.cpp
index cc80d5bd138980e..b7b5e86953d589b 100644
--- a/cloud/src/meta-service/meta_service_job.cpp
+++ b/cloud/src/meta-service/meta_service_job.cpp
@@ -876,6 +876,14 @@ void process_compaction_job(MetaServiceCode& code, std::string& msg, std::string
 
     txn->remove(rs_start, rs_end);
 
+    LOG_INFO("cloud process compaction job txn remove meta rowset key")
+            .tag("instance_id", instance_id)
+            .tag("tablet_id", tablet_id)
+            .tag("start_version", start)
+            .tag("end_version", end + 1)
+            .tag("rs_start key", hex(rs_start))
+            .tag("rs_end key", hex(rs_end));
+
     TEST_SYNC_POINT_CALLBACK("process_compaction_job::loop_input_done", &num_rowsets);
 
     if (num_rowsets < 1) {
diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp
index 04476704bd36c70..ba67711866bb0ac 100644
--- a/cloud/src/recycler/recycler.cpp
+++ b/cloud/src/recycler/recycler.cpp
@@ -2893,9 +2893,9 @@ int InstanceRecycler::recycle_expired_stage_objects() {
     int ret = 0;
     for (const auto& stage : instance_info_.stages()) {
         std::stringstream ss;
-        ss << "instance_id=" << instance_id_ << ", stage_id=" << stage.stage_id()
-           << ", user_name=" << stage.mysql_user_name().at(0)
-           << ", user_id=" << stage.mysql_user_id().at(0)
+        ss << "instance_id=" << instance_id_ << ", stage_id=" << stage.stage_id() << ", user_name="
+           << (stage.mysql_user_name().empty() ? "null" : stage.mysql_user_name().at(0))
+           << ", user_id=" << (stage.mysql_user_id().empty() ? "null" : stage.mysql_user_id().at(0))
            << ", prefix=" << stage.obj_info().prefix();
 
         if (stopped()) break;

From a7645051e88fce76f45c0e75a6c6d62505e13d64 Mon Sep 17 00:00:00 2001
From: zhangstar333 <zhangsida@selectdb.com>
Date: Wed, 8 Jan 2025 09:58:50 +0800
Subject: [PATCH 032/140] [chore](session) remove some unused session variables
 (#45585)

---
 be/src/runtime/runtime_state.h                | 14 -------
 .../org/apache/doris/qe/SessionVariable.java  | 37 +------------------
 gensrc/thrift/PaloInternalService.thrift      |  6 +--
 .../query/test_nested_type_with_resize.groovy |  8 ++--
 .../mv/variant/variant_mv.groovy              |  2 +-
 .../join/test_partitioned_hash_join.groovy    |  2 +-
 regression-test/suites/variant_p0/load.groovy |  2 +-
 .../suites/variant_p0/nested.groovy           |  4 +-
 .../variant_p0/test_sub_path_pruning.groovy   |  2 +-
 9 files changed, 15 insertions(+), 62 deletions(-)

diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h
index 3cc3d97b5d22c5a..e365c0608b71e0b 100644
--- a/be/src/runtime/runtime_state.h
+++ b/be/src/runtime/runtime_state.h
@@ -403,20 +403,6 @@ class RuntimeState {
 
     bool enable_page_cache() const;
 
-    int partitioned_hash_join_rows_threshold() const {
-        if (!_query_options.__isset.partitioned_hash_join_rows_threshold) {
-            return 0;
-        }
-        return _query_options.partitioned_hash_join_rows_threshold;
-    }
-
-    int partitioned_hash_agg_rows_threshold() const {
-        if (!_query_options.__isset.partitioned_hash_agg_rows_threshold) {
-            return 0;
-        }
-        return _query_options.partitioned_hash_agg_rows_threshold;
-    }
-
     const std::vector<TTabletCommitInfo>& tablet_commit_infos() const {
         return _tablet_commit_infos;
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 0b5a6f785ec4c24..5e663f0112699ac 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -128,7 +128,6 @@ public class SessionVariable implements Serializable, Writable {
     public static final String TIME_ZONE = "time_zone";
     public static final String SQL_SAFE_UPDATES = "sql_safe_updates";
     public static final String NET_BUFFER_LENGTH = "net_buffer_length";
-    public static final String CODEGEN_LEVEL = "codegen_level";
     public static final String HAVE_QUERY_CACHE =  "have_query_cache";
     // mem limit can't smaller than bufferpool's default page size
     public static final int MIN_EXEC_MEM_LIMIT = 2097152;
@@ -404,9 +403,6 @@ public class SessionVariable implements Serializable, Writable {
 
     public static final String INTERNAL_SESSION = "internal_session";
 
-    public static final String PARTITIONED_HASH_JOIN_ROWS_THRESHOLD = "partitioned_hash_join_rows_threshold";
-    public static final String PARTITIONED_HASH_AGG_ROWS_THRESHOLD = "partitioned_hash_agg_rows_threshold";
-
     public static final String PARTITION_PRUNING_EXPAND_THRESHOLD = "partition_pruning_expand_threshold";
 
     public static final String ENABLE_SHARE_HASH_TABLE_FOR_BROADCAST_JOIN
@@ -937,10 +933,6 @@ public class SessionVariable implements Serializable, Writable {
     @VariableMgr.VarAttr(name = NET_BUFFER_LENGTH, flag = VariableMgr.READ_ONLY)
     public int netBufferLength = 16384;
 
-    // if true, need report to coordinator when plan fragment execute successfully.
-    @VariableMgr.VarAttr(name = CODEGEN_LEVEL)
-    public int codegenLevel = 0;
-
     @VariableMgr.VarAttr(name = HAVE_QUERY_CACHE, flag = VariableMgr.READ_ONLY)
     public boolean haveQueryCache = false;
 
@@ -1535,7 +1527,8 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) {
     @VariableMgr.VarAttr(name = ENABLE_COMMON_EXPR_PUSHDOWN, fuzzy = true)
     public boolean enableCommonExprPushdown = true;
 
-    @VariableMgr.VarAttr(name = ENABLE_LOCAL_EXCHANGE, fuzzy = true, varType = VariableAnnotation.DEPRECATED)
+    @VariableMgr.VarAttr(name = ENABLE_LOCAL_EXCHANGE, fuzzy = true, flag = VariableMgr.INVISIBLE,
+            varType = VariableAnnotation.DEPRECATED)
     public boolean enableLocalExchange = true;
 
     /**
@@ -1611,14 +1604,6 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) {
     @VariableMgr.VarAttr(name = INTERNAL_SESSION)
     public boolean internalSession = false;
 
-    // Use partitioned hash join if build side row count >= the threshold . 0 - the threshold is not set.
-    @VariableMgr.VarAttr(name = PARTITIONED_HASH_JOIN_ROWS_THRESHOLD, fuzzy = true)
-    public int partitionedHashJoinRowsThreshold = 0;
-
-    // Use partitioned hash join if build side row count >= the threshold . 0 - the threshold is not set.
-    @VariableMgr.VarAttr(name = PARTITIONED_HASH_AGG_ROWS_THRESHOLD, fuzzy = true)
-    public int partitionedHashAggRowsThreshold = 0;
-
     @VariableMgr.VarAttr(name = PARTITION_PRUNING_EXPAND_THRESHOLD, fuzzy = true)
     public int partitionPruningExpandThreshold = 10;
 
@@ -2444,8 +2429,6 @@ public void initFuzzyModeVariables() {
         // this.disableJoinReorder = random.nextBoolean();
         this.enableCommonExpPushDownForInvertedIndex = random.nextBoolean();
         this.disableStreamPreaggregations = random.nextBoolean();
-        this.partitionedHashJoinRowsThreshold = random.nextBoolean() ? 8 : 1048576;
-        this.partitionedHashAggRowsThreshold = random.nextBoolean() ? 8 : 1048576;
         this.enableShareHashTableForBroadcastJoin = random.nextBoolean();
         // this.enableHashJoinEarlyStartProbe = random.nextBoolean();
         this.enableParallelResultSink = random.nextBoolean();
@@ -2866,10 +2849,6 @@ public int getNetBufferLength() {
         return netBufferLength;
     }
 
-    public int getCodegenLevel() {
-        return codegenLevel;
-    }
-
     public boolean getHaveQueryCache() {
         return haveQueryCache;
     }
@@ -3145,14 +3124,6 @@ public void setQueryCacheEntryMaxRows(long queryCacheEntryMaxRows) {
         this.queryCacheEntryMaxRows = queryCacheEntryMaxRows;
     }
 
-    public int getPartitionedHashJoinRowsThreshold() {
-        return partitionedHashJoinRowsThreshold;
-    }
-
-    public void setPartitionedHashJoinRowsThreshold(int threshold) {
-        this.partitionedHashJoinRowsThreshold = threshold;
-    }
-
     // Serialize to thrift object
     public boolean getForwardToMaster() {
         return forwardToMaster;
@@ -3951,7 +3922,6 @@ public TQueryOptions toThrift() {
             // we should modify BE in the future.
             tResult.setIsReportSuccess(true);
         }
-        tResult.setCodegenLevel(codegenLevel);
         tResult.setBeExecVersion(Config.be_exec_version);
         tResult.setEnableLocalShuffle(enableLocalShuffle);
         tResult.setParallelInstance(getParallelExecInstanceNum());
@@ -4004,9 +3974,6 @@ public TQueryOptions toThrift() {
 
         tResult.setSkipDeleteBitmap(skipDeleteBitmap);
 
-        tResult.setPartitionedHashJoinRowsThreshold(partitionedHashJoinRowsThreshold);
-        tResult.setPartitionedHashAggRowsThreshold(partitionedHashAggRowsThreshold);
-
         tResult.setExternalSortBytesThreshold(externalSortBytesThreshold);
 
         tResult.setExternalAggBytesThreshold(0); // disable for now
diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift
index c219daef5d1ad2b..70db19bfa335817 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -103,7 +103,7 @@ struct TQueryOptions {
   13: optional bool abort_on_default_limit_exceeded = 0
   14: optional i32 query_timeout = 3600
   15: optional bool is_report_success = 0
-  16: optional i32 codegen_level = 0
+  16: optional i32 codegen_level = 0 // Deprecated
   // INT64::MAX
   17: optional i64 kudu_latest_observed_ts = 9223372036854775807 // Deprecated
   18: optional TQueryType query_type = TQueryType.SELECT
@@ -181,7 +181,7 @@ struct TQueryOptions {
 
   52: optional i32 be_exec_version = 0
 
-  53: optional i32 partitioned_hash_join_rows_threshold = 0
+  53: optional i32 partitioned_hash_join_rows_threshold = 0 // deprecated
 
   54: optional bool enable_share_hash_table_for_broadcast_join
 
@@ -197,7 +197,7 @@ struct TQueryOptions {
   59: optional i64 external_sort_bytes_threshold = 0
 
   // deprecated
-  60: optional i32 partitioned_hash_agg_rows_threshold = 0
+  60: optional i32 partitioned_hash_agg_rows_threshold = 0 // deprecated
 
   61: optional bool enable_file_cache = false
 
diff --git a/regression-test/suites/datatype_p0/nested_types/query/test_nested_type_with_resize.groovy b/regression-test/suites/datatype_p0/nested_types/query/test_nested_type_with_resize.groovy
index 2a599c1cc1ad20b..c68107dcd37a917 100644
--- a/regression-test/suites/datatype_p0/nested_types/query/test_nested_type_with_resize.groovy
+++ b/regression-test/suites/datatype_p0/nested_types/query/test_nested_type_with_resize.groovy
@@ -65,9 +65,9 @@ suite("test_nested_type_with_resize") {
         }
     }
 
-    order_qt_sql """ /*set ShuffleSendBytes=0|ShuffleSendRows=0|FuzzyVariables=batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=3,parallel_pipeline_task_num=5,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=true,parallel_scan_max_scanners_count=48,parallel_scan_min_rows_per_scanner=16384,parallel_prepare_threshold=13,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=12,enable_parallel_result_sink=true,sort_phase_num=0,rewrite_or_to_in_predicate_threshold=100000,enable_function_pushdown=false,enable_common_expr_pushdown=true,enable_local_exchange=false,partitioned_hash_join_rows_threshold=1048576,partitioned_hash_agg_rows_threshold=1048576,partition_pruning_expand_threshold=10,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_delete_sub_predicate_v2=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5*/ select col4 from test_array_resize order by col1,col2,col3 limit 10; """
-    order_qt_sql """ /*set ShuffleSendBytes=0|ShuffleSendRows=0|FuzzyVariables=batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=3,parallel_pipeline_task_num=5,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=true,parallel_scan_max_scanners_count=48,parallel_scan_min_rows_per_scanner=16384,parallel_prepare_threshold=13,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=12,enable_parallel_result_sink=true,sort_phase_num=0,rewrite_or_to_in_predicate_threshold=100000,enable_function_pushdown=false,enable_common_expr_pushdown=true,enable_local_exchange=false,partitioned_hash_join_rows_threshold=1048576,partitioned_hash_agg_rows_threshold=1048576,partition_pruning_expand_threshold=10,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_delete_sub_predicate_v2=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5*/ select col22 from test_array_resize order by col1,col2,col3 limit 10; """
-    order_qt_sql """ /*set ShuffleSendBytes=0|ShuffleSendRows=0|FuzzyVariables=batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=3,parallel_pipeline_task_num=5,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=true,parallel_scan_max_scanners_count=48,parallel_scan_min_rows_per_scanner=16384,parallel_prepare_threshold=13,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=12,enable_parallel_result_sink=true,sort_phase_num=0,rewrite_or_to_in_predicate_threshold=100000,enable_function_pushdown=false,enable_common_expr_pushdown=true,enable_local_exchange=false,partitioned_hash_join_rows_threshold=1048576,partitioned_hash_agg_rows_threshold=1048576,partition_pruning_expand_threshold=10,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_delete_sub_predicate_v2=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5*/ select col32 from test_array_resize order by col1,col2,col3 limit 10; """
-    order_qt_sql """ /*set ShuffleSendBytes=0|ShuffleSendRows=0|FuzzyVariables=batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=3,parallel_pipeline_task_num=5,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=true,parallel_scan_max_scanners_count=48,parallel_scan_min_rows_per_scanner=16384,parallel_prepare_threshold=13,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=12,enable_parallel_result_sink=true,sort_phase_num=0,rewrite_or_to_in_predicate_threshold=100000,enable_function_pushdown=false,enable_common_expr_pushdown=true,enable_local_exchange=false,partitioned_hash_join_rows_threshold=1048576,partitioned_hash_agg_rows_threshold=1048576,partition_pruning_expand_threshold=10,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_delete_sub_predicate_v2=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5*/ select col74 from test_array_resize order by col1,col2,col3 limit 10; """
+    order_qt_sql """ /*set ShuffleSendBytes=0|ShuffleSendRows=0|FuzzyVariables=batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=3,parallel_pipeline_task_num=5,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=true,parallel_scan_max_scanners_count=48,parallel_scan_min_rows_per_scanner=16384,parallel_prepare_threshold=13,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=12,enable_parallel_result_sink=true,sort_phase_num=0,rewrite_or_to_in_predicate_threshold=100000,enable_function_pushdown=false,enable_common_expr_pushdown=true,enable_local_exchange=false,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_delete_sub_predicate_v2=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5*/ select col4 from test_array_resize order by col1,col2,col3 limit 10; """
+    order_qt_sql """ /*set ShuffleSendBytes=0|ShuffleSendRows=0|FuzzyVariables=batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=3,parallel_pipeline_task_num=5,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=true,parallel_scan_max_scanners_count=48,parallel_scan_min_rows_per_scanner=16384,parallel_prepare_threshold=13,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=12,enable_parallel_result_sink=true,sort_phase_num=0,rewrite_or_to_in_predicate_threshold=100000,enable_function_pushdown=false,enable_common_expr_pushdown=true,enable_local_exchange=false,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_delete_sub_predicate_v2=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5*/ select col22 from test_array_resize order by col1,col2,col3 limit 10; """
+    order_qt_sql """ /*set ShuffleSendBytes=0|ShuffleSendRows=0|FuzzyVariables=batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=3,parallel_pipeline_task_num=5,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=true,parallel_scan_max_scanners_count=48,parallel_scan_min_rows_per_scanner=16384,parallel_prepare_threshold=13,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=12,enable_parallel_result_sink=true,sort_phase_num=0,rewrite_or_to_in_predicate_threshold=100000,enable_function_pushdown=false,enable_common_expr_pushdown=true,enable_local_exchange=false,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_delete_sub_predicate_v2=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5*/ select col32 from test_array_resize order by col1,col2,col3 limit 10; """
+    order_qt_sql """ /*set ShuffleSendBytes=0|ShuffleSendRows=0|FuzzyVariables=batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=3,parallel_pipeline_task_num=5,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=true,parallel_scan_max_scanners_count=48,parallel_scan_min_rows_per_scanner=16384,parallel_prepare_threshold=13,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=12,enable_parallel_result_sink=true,sort_phase_num=0,rewrite_or_to_in_predicate_threshold=100000,enable_function_pushdown=false,enable_common_expr_pushdown=true,enable_local_exchange=false,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_delete_sub_predicate_v2=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5*/ select col74 from test_array_resize order by col1,col2,col3 limit 10; """
 
 }
diff --git a/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy b/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy
index fae32e914ed031e..942c4ce88dee7a9 100644
--- a/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy
@@ -574,7 +574,7 @@ suite("variant_mv") {
     where g2.actor['id'] > 34259289;
     """
     def query3_6 = """
-    SELECT  /*+SET_VAR(batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=3,parallel_pipeline_task_num=0,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=true,parallel_scan_max_scanners_count=32,parallel_scan_min_rows_per_scanner=64,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=1,enable_parallel_result_sink=false,enable_nereids_planner=true,rewrite_or_to_in_predicate_threshold=100000,enable_function_pushdown=false,enable_common_expr_pushdown=false,enable_local_exchange=true,partitioned_hash_join_rows_threshold=8,partitioned_hash_agg_rows_threshold=8,partition_pruning_expand_threshold=10,enable_share_hash_table_for_broadcast_join=true,enable_two_phase_read_opt=true,enable_common_expr_pushdown_for_inverted_index=false,enable_delete_sub_predicate_v2=false,min_revocable_mem=33554432,fetch_remote_schema_timeout_seconds=120,max_fetch_remote_schema_tablet_count=512,enable_join_spill=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5) */
+    SELECT  /*+SET_VAR(batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=3,parallel_pipeline_task_num=0,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=true,parallel_scan_max_scanners_count=32,parallel_scan_min_rows_per_scanner=64,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=1,enable_parallel_result_sink=false,enable_nereids_planner=true,rewrite_or_to_in_predicate_threshold=100000,enable_function_pushdown=false,enable_common_expr_pushdown=false,enable_local_exchange=true,enable_share_hash_table_for_broadcast_join=true,enable_two_phase_read_opt=true,enable_common_expr_pushdown_for_inverted_index=false,enable_delete_sub_predicate_v2=false,min_revocable_mem=33554432,fetch_remote_schema_timeout_seconds=120,max_fetch_remote_schema_tablet_count=512,enable_join_spill=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5) */
     g1.id,
     g2.type,
     floor(cast(g1.actor['id'] as int) + 100.5),
diff --git a/regression-test/suites/query_p0/join/test_partitioned_hash_join.groovy b/regression-test/suites/query_p0/join/test_partitioned_hash_join.groovy
index 676cdd06274a686..33a2595de2d9d9f 100644
--- a/regression-test/suites/query_p0/join/test_partitioned_hash_join.groovy
+++ b/regression-test/suites/query_p0/join/test_partitioned_hash_join.groovy
@@ -65,7 +65,7 @@ suite("test_partitioned_hash_join", "query,p0,arrow_flight_sql") {
 
     qt_partitioned_hash_join1 """
         select
-            /*+SET_VAR(disable_join_reorder=true,experimental_enable_pipeline_engine=false, parallel_fragment_exec_instance_num=1, partitioned_hash_join_rows_threshold = 1)*/
+            /*+SET_VAR(disable_join_reorder=true,experimental_enable_pipeline_engine=false, parallel_fragment_exec_instance_num=1)*/
             kl1
         from
             test_partitioned_hash_join_l
diff --git a/regression-test/suites/variant_p0/load.groovy b/regression-test/suites/variant_p0/load.groovy
index cd5e9ee523d9226..b679a3ad9d96579 100644
--- a/regression-test/suites/variant_p0/load.groovy
+++ b/regression-test/suites/variant_p0/load.groovy
@@ -290,7 +290,7 @@ suite("regression_test_variant", "p0"){
         sql """insert into ${table_name} values (5, '{"i" : 1}'), (1, '{"a" : 1}')"""
         sql """insert into ${table_name} values (6, '{"j" : 1}'), (1, '{"a" : 1}')"""
         sql """insert into ${table_name} values (6, '{"k" : 1}'), (1, '{"a" : 1}')"""
-        sql "select /*+SET_VAR(batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=1,parallel_pipeline_task_num=4,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=true,parallel_scan_max_scanners_count=16,parallel_scan_min_rows_per_scanner=128,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=2,enable_parallel_result_sink=false,sort_phase_num=0,enable_nereids_planner=true,rewrite_or_to_in_predicate_threshold=2,enable_function_pushdown=true,enable_common_expr_pushdown=true,enable_local_exchange=true,partitioned_hash_join_rows_threshold=1048576,partitioned_hash_agg_rows_threshold=8,partition_pruning_expand_threshold=10,enable_share_hash_table_for_broadcast_join=true,enable_two_phase_read_opt=true,enable_common_expr_pushdown_for_inverted_index=false,enable_delete_sub_predicate_v2=true,min_revocable_mem=33554432,fetch_remote_schema_timeout_seconds=120,max_fetch_remote_schema_tablet_count=512,enable_join_spill=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5) */ * from ${table_name}"
+        sql "select /*+SET_VAR(batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=1,parallel_pipeline_task_num=4,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=true,parallel_scan_max_scanners_count=16,parallel_scan_min_rows_per_scanner=128,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=2,enable_parallel_result_sink=false,sort_phase_num=0,enable_nereids_planner=true,rewrite_or_to_in_predicate_threshold=2,enable_function_pushdown=true,enable_common_expr_pushdown=true,enable_local_exchange=true,partition_pruning_expand_threshold=10,enable_share_hash_table_for_broadcast_join=true,enable_two_phase_read_opt=true,enable_common_expr_pushdown_for_inverted_index=false,enable_delete_sub_predicate_v2=true,min_revocable_mem=33554432,fetch_remote_schema_timeout_seconds=120,max_fetch_remote_schema_tablet_count=512,enable_join_spill=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5) */ * from ${table_name}"
         qt_sql_36_1 "select cast(v['a'] as int), cast(v['b'] as int), cast(v['c'] as int) from ${table_name} order by k limit 10"
         sql "DELETE FROM ${table_name} WHERE k=1"
         sql "select * from ${table_name}"
diff --git a/regression-test/suites/variant_p0/nested.groovy b/regression-test/suites/variant_p0/nested.groovy
index 7235386960fb616..8eaec744b029eb3 100644
--- a/regression-test/suites/variant_p0/nested.groovy
+++ b/regression-test/suites/variant_p0/nested.groovy
@@ -88,7 +88,7 @@ suite("regression_test_variant_nested", "p0"){
 
         qt_sql """select  /*+SET_VAR(batch_size=1024,broker_load_batch_size=16352,disable_streaming_preaggregations=true,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_
 parallel_pipeline_task_num=7,parallel_fragment_exec_instance_num=4,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=false,parallel_scan_max_scanners_count=16
-,parallel_scan_min_rows_per_scanner=128,enable_fold_constant_by_be=false,enable_rewrite_element_at_to_slot=true,runtime_filter_type=2,enable_parallel_result_sink=true,sort_phase_num=0,enable_nereids_planner=true,rewrite_or_to_in_predicate_threshold=2,enable_function_pushdown=true,enable_common_expr_pushdown=false,enable_local_exchange=true,partitioned_hash_join_rows_threshold=8,partitioned_hash_agg_rows_threshold=8,partition_pruning_expand_threshold=10,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_common_expr_pushdown_for_inverted_index=true,enable_delete_sub_predicate_v2=true,min_revocable_mem=1,fetch_remote_schema_timeout_seconds=120,max_fetch_remote_schema_tablet_count=512,enable_join_spill=true,enable_sort_spill=true,enable_agg_spill=true,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5) */ * from var_nested where v['k2'] = 'some' order by k limit 10"""
+,parallel_scan_min_rows_per_scanner=128,enable_fold_constant_by_be=false,enable_rewrite_element_at_to_slot=true,runtime_filter_type=2,enable_parallel_result_sink=true,sort_phase_num=0,enable_nereids_planner=true,rewrite_or_to_in_predicate_threshold=2,enable_function_pushdown=true,enable_common_expr_pushdown=false,enable_local_exchange=true,partition_pruning_expand_threshold=10,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_common_expr_pushdown_for_inverted_index=true,enable_delete_sub_predicate_v2=true,min_revocable_mem=1,fetch_remote_schema_timeout_seconds=120,max_fetch_remote_schema_tablet_count=512,enable_join_spill=true,enable_sort_spill=true,enable_agg_spill=true,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5) */ * from var_nested where v['k2'] = 'some' order by k limit 10"""
         qt_sql """select * from var_nested where v['k2'] = 'some'  and array_contains(cast(v['nested1']['nested2']['a'] as array<tinyint>), 10) order by k limit 1;"""
 
         sql """INSERT INTO var_nested SELECT *, '{"k1":1, "k2": "some", "k3" : [1234], "k4" : 1.10000, "k5" : [[123]], "nested1" : {"nested2" : [{"a" : 10, "b" : 1.1, "c" : "1111"}]}}' FROM numbers("number" = "4096") where number > 1024 limit 1024;"""
@@ -117,7 +117,7 @@ parallel_pipeline_task_num=7,parallel_fragment_exec_instance_num=4,profile_level
                 properties("replication_num" = "1", "disable_auto_compaction" = "false", "enable_unique_key_merge_on_write" = "true", "variant_enable_flatten_nested" = "true");
             """
         sql """insert into var_nested2 select * from var_nested order by k limit 1024"""
-        qt_sql """select  /*+SET_VAR(batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=true,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=5,parallel_pipeline_task_num=1,profile_level=1,enable_pipeline_engine=false,enable_parallel_scan=true,parallel_scan_max_scanners_count=48,parallel_scan_min_rows_per_scanner=16384,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=12,enable_parallel_result_sink=false,enable_nereids_planner=true,rewrite_or_to_in_predicate_threshold=2,enable_function_pushdown=true,enable_common_expr_pushdown=false,enable_local_exchange=false,partitioned_hash_join_rows_threshold=1048576,partitioned_hash_agg_rows_threshold=8,partition_pruning_expand_threshold=10,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_common_expr_pushdown_for_inverted_index=true,enable_delete_sub_predicate_v2=true,min_revocable_mem=33554432,fetch_remote_schema_timeout_seconds=120,max_fetch_remote_schema_tablet_count=512,enable_join_spill=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5) */  * from var_nested2 order by k limit 10;"""
+        qt_sql """select  /*+SET_VAR(batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=true,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=5,parallel_pipeline_task_num=1,profile_level=1,enable_pipeline_engine=false,enable_parallel_scan=true,parallel_scan_max_scanners_count=48,parallel_scan_min_rows_per_scanner=16384,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=12,enable_parallel_result_sink=false,enable_nereids_planner=true,rewrite_or_to_in_predicate_threshold=2,enable_function_pushdown=true,enable_common_expr_pushdown=false,enable_local_exchange=false,partition_pruning_expand_threshold=10,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_common_expr_pushdown_for_inverted_index=true,enable_delete_sub_predicate_v2=true,min_revocable_mem=33554432,fetch_remote_schema_timeout_seconds=120,max_fetch_remote_schema_tablet_count=512,enable_join_spill=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5) */  * from var_nested2 order by k limit 10;"""
         qt_sql """select v['nested'] from var_nested2 where k < 10 order by k limit 10;"""
         // 0. nomal explode variant array
         order_qt_explode_sql """select count(),cast(vv['xx'] as int) from var_nested lateral view explode_variant_array(v['nested']) tmp as vv where vv['xx'] = 10 group by cast(vv['xx'] as int)"""
diff --git a/regression-test/suites/variant_p0/test_sub_path_pruning.groovy b/regression-test/suites/variant_p0/test_sub_path_pruning.groovy
index f09f4713ad29e52..c1547a0a6e039d1 100644
--- a/regression-test/suites/variant_p0/test_sub_path_pruning.groovy
+++ b/regression-test/suites/variant_p0/test_sub_path_pruning.groovy
@@ -139,7 +139,7 @@ suite("variant_sub_path_pruning", "variant_type"){
 
     // two children
     order_qt_sql """
-        select  /*+SET_VAR(batch_size=50,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=6,parallel_pipeline_task_num=2,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=false,parallel_scan_max_scanners_count=16,parallel_scan_min_rows_per_scanner=128,enable_fold_constant_by_be=false,enable_rewrite_element_at_to_slot=true,runtime_filter_type=2,enable_nereids_planner=true,rewrite_or_to_in_predicate_threshold=2,enable_function_pushdown=true,enable_common_expr_pushdown=true,enable_local_exchange=false,partitioned_hash_join_rows_threshold=1048576,partitioned_hash_agg_rows_threshold=8,partition_pruning_expand_threshold=10,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_delete_sub_predicate_v2=true,min_revocable_mem=33554432,fetch_remote_schema_timeout_seconds=120,enable_join_spill=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5) */ dt['a'] as c1 from pruning_test union all select dt['a'] as c1 from pruning_test;
+        select  /*+SET_VAR(batch_size=50,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=6,parallel_pipeline_task_num=2,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=false,parallel_scan_max_scanners_count=16,parallel_scan_min_rows_per_scanner=128,enable_fold_constant_by_be=false,enable_rewrite_element_at_to_slot=true,runtime_filter_type=2,enable_nereids_planner=true,rewrite_or_to_in_predicate_threshold=2,enable_function_pushdown=true,enable_common_expr_pushdown=true,enable_local_exchange=false,partition_pruning_expand_threshold=10,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_delete_sub_predicate_v2=true,min_revocable_mem=33554432,fetch_remote_schema_timeout_seconds=120,enable_join_spill=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5) */ dt['a'] as c1 from pruning_test union all select dt['a'] as c1 from pruning_test;
         """
     order_qt_sql """select c1['a'] from (select dt as c1 from pruning_test union all select dt as c1 from pruning_test) v1;"""
     order_qt_sql """select c1['b'] from (select dt['a'] as c1 from pruning_test union all select dt['a'] as c1 from pruning_test) v1;"""

From 630b0ff5a83dad69ab8acd0da261a9dc6b2b16cb Mon Sep 17 00:00:00 2001
From: HappenLee <happenlee@selectdb.com>
Date: Wed, 8 Jan 2025 10:00:43 +0800
Subject: [PATCH 033/140] [Fix](func) fix group_array_intersect result error in
 all arr contain null (#46521)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Data：
```
CREATE TABLE `group_array_intersect_test_not_null` (
            `id` int(11) NULL COMMENT ""
            , `c_array_int` ARRAY<int(11)> NOT NULL COMMENT ""
            , `c_array_float` ARRAY<float> NOT NULL COMMENT ""
            , `c_array_string` ARRAY<string> NOT NULL COMMENT ""
        ) ENGINE=OLAP
        DUPLICATE KEY(`id`)
        COMMENT "OLAP"
        DISTRIBUTED BY HASH(`id`) BUCKETS 1
        PROPERTIES (
        "replication_allocation" = "tag.location.default: 1",
        "in_memory" = "false",
        "storage_format" = "V2"
        );

    sql """INSERT INTO `group_array_intersect_test_not_null`(id, c_array_int, c_array_float, c_array_string) VALUES (3, [], [], []);"""
    sql """INSERT INTO `group_array_intersect_test_not_null`(id, c_array_int, c_array_float, c_array_string) VALUES (4, [6, 7], [6.6, 7.7], ['a']);"""
    sql """INSERT INTO `group_array_intersect_test_not_null`(id, c_array_int, c_array_float, c_array_string) VALUES (5, [null], [null], ['x', 'y']);"""

```

query：
```
select array_sort(group_array_intersect(c_array_float)) from group_array_intersect_test_not_null where id between 1 and 5;
```

result should be [] not [null]
---
 be/src/exprs/hybrid_set.h                             |  1 +
 .../aggregate_function_group_array_intersect.h        | 11 ++++-------
 .../data/query_p0/aggregate/group_array_intersect.out |  3 +++
 .../query_p0/aggregate/group_array_intersect.groovy   |  1 +
 4 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/be/src/exprs/hybrid_set.h b/be/src/exprs/hybrid_set.h
index dbf31a54772ea3b..055b73626598bd1 100644
--- a/be/src/exprs/hybrid_set.h
+++ b/be/src/exprs/hybrid_set.h
@@ -202,6 +202,7 @@ class HybridSetBase : public RuntimeFilterFuncBase {
         _contains_null |= set->_contains_null;
     }
 
+    bool empty() { return !_contains_null && size() == 0; }
     virtual int size() = 0;
     virtual bool find(const void* data) const = 0;
     // use in vectorize execute engine
diff --git a/be/src/vec/aggregate_functions/aggregate_function_group_array_intersect.h b/be/src/vec/aggregate_functions/aggregate_function_group_array_intersect.h
index 1bb6a7a170486cb..a93c74c1fe2b36d 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_group_array_intersect.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_group_array_intersect.h
@@ -124,7 +124,7 @@ struct AggregateFunctionGroupArrayIntersectData {
                 set->insert(src_data);
             }
             init = true;
-        } else if (set->size() != 0 || set->contain_null()) {
+        } else if (!set->empty()) {
             Set new_set = std::make_unique<NullableNumericOrDateSetType>();
 
             for (size_t i = 0; i < arr_size; ++i) {
@@ -214,10 +214,7 @@ class AggregateFunctionGroupArrayIntersect
                 it->next();
             }
             init = true;
-            return;
-        }
-
-        if (set->size() != 0) {
+        } else if (!set->empty()) {
             auto create_new_set = [](auto& lhs_val, auto& rhs_val) {
                 typename State::Set new_set =
                         std::make_unique<typename State::NullableNumericOrDateSetType>();
@@ -420,7 +417,7 @@ class AggregateFunctionGroupArrayIntersectGeneric
                 set->insert((void*)src.data, src.size);
             }
             init = true;
-        } else if (set->size() != 0 || set->contain_null()) {
+        } else if (!set->empty()) {
             typename State::Set new_set = std::make_unique<NullableStringSet>();
 
             for (size_t i = 0; i < arr_size; ++i) {
@@ -454,7 +451,7 @@ class AggregateFunctionGroupArrayIntersectGeneric
                 it->next();
             }
             init = true;
-        } else if (set->size() != 0) {
+        } else if (!set->empty()) {
             auto create_new_set = [](auto& lhs_val, auto& rhs_val) {
                 typename State::Set new_set = std::make_unique<NullableStringSet>();
                 HybridSetBase::IteratorBase* it = lhs_val->begin();
diff --git a/regression-test/data/query_p0/aggregate/group_array_intersect.out b/regression-test/data/query_p0/aggregate/group_array_intersect.out
index c58ede1043e4221..7778d24b7032ac7 100644
--- a/regression-test/data/query_p0/aggregate/group_array_intersect.out
+++ b/regression-test/data/query_p0/aggregate/group_array_intersect.out
@@ -91,3 +91,6 @@
 -- !notnull_6 --
 []
 
+-- !notnull_7 --
+[]
+
diff --git a/regression-test/suites/query_p0/aggregate/group_array_intersect.groovy b/regression-test/suites/query_p0/aggregate/group_array_intersect.groovy
index 8e0ecbe3c21bc27..6141dea3da8178f 100644
--- a/regression-test/suites/query_p0/aggregate/group_array_intersect.groovy
+++ b/regression-test/suites/query_p0/aggregate/group_array_intersect.groovy
@@ -103,4 +103,5 @@ suite("group_array_intersect") {
     qt_notnull_4 """select array_sort(group_array_intersect(c_array_string)) from group_array_intersect_test_not_null where id between 1 and 2;"""
     qt_notnull_5 """select array_sort(group_array_intersect(c_array_int)), array_sort(group_array_intersect(c_array_float)), array_sort(group_array_intersect(c_array_string)) from group_array_intersect_test_not_null where id between 3 and 4;"""
     qt_notnull_6 """select array_sort(group_array_intersect(c_array_string)) from group_array_intersect_test_not_null where id between 1 and 5;"""
+    qt_notnull_7 """select array_sort(group_array_intersect(c_array_float)) from group_array_intersect_test_not_null where id between 1 and 5;"""
 }

From 6de9d27edb388e61b6a48848bac9a16390ca8c08 Mon Sep 17 00:00:00 2001
From: feiniaofeiafei <moailing@selectdb.com>
Date: Wed, 8 Jan 2025 10:03:02 +0800
Subject: [PATCH 034/140] [fix](nereids) fix bug of LogicalRepeat computing
 uniform datatrait (#46352)

### What problem does this PR solve?

Related PR: #43391

Problem Summary:
repeat's uniform is not right, remove it temprarily
---
 .../nereids/trees/plans/logical/LogicalRepeat.java     |  3 ++-
 .../apache/doris/nereids/properties/DataTraitTest.java |  3 +--
 .../eliminate_group_by_key_by_uniform.out              | 10 ++++++++++
 .../eliminate_group_by_key_by_uniform.groovy           |  3 +++
 4 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java
index 10695f655508afb..005ac35cf9913e8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java
@@ -190,7 +190,8 @@ public void computeUnique(DataTrait.Builder builder) {
 
     @Override
     public void computeUniform(DataTrait.Builder builder) {
-        builder.addUniformSlot(child(0).getLogicalProperties().getTrait());
+        // don't generate uniform slot
+        // TODO: this need be supported later
     }
 
     @Override
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/DataTraitTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/DataTraitTest.java
index 04e8ec7fc3240ac..246495c659d9c62 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/DataTraitTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/DataTraitTest.java
@@ -242,8 +242,7 @@ void testAgg() {
                 .rewrite()
                 .getPlan();
         System.out.println(plan.getLogicalProperties().getTrait());
-        Assertions.assertTrue(plan.getLogicalProperties()
-                .getTrait().isUniform(plan.getOutput().get(0)));
+        // Assertions.assertTrue(plan.getLogicalProperties().getTrait().isUniform(plan.getOutput().get(0)));
     }
 
     @Test
diff --git a/regression-test/data/nereids_rules_p0/eliminate_gby_key/eliminate_group_by_key_by_uniform.out b/regression-test/data/nereids_rules_p0/eliminate_gby_key/eliminate_group_by_key_by_uniform.out
index 94cebafa4e9aad9..625b8e754886695 100644
--- a/regression-test/data/nereids_rules_p0/eliminate_gby_key/eliminate_group_by_key_by_uniform.out
+++ b/regression-test/data/nereids_rules_p0/eliminate_gby_key/eliminate_group_by_key_by_uniform.out
@@ -267,3 +267,13 @@ cherry	3
 
 -- !right_anti_right_side --
 
+-- !grouping --
+\N	\N
+\N	1
+\N	2
+\N	3
+1	\N
+1	1
+1	2
+1	3
+
diff --git a/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_group_by_key_by_uniform.groovy b/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_group_by_key_by_uniform.groovy
index 4375aac316d5d19..399be417c8baf67 100644
--- a/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_group_by_key_by_uniform.groovy
+++ b/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_group_by_key_by_uniform.groovy
@@ -218,4 +218,7 @@ suite("eliminate_group_by_key_by_uniform") {
     qt_left_anti_left_side "select t1.b from test1 t1 left anti join (select * from test2 where b=105)  t2 on t1.a=t2.a where t1.b=1 group by t1.b,t1.a order by 1;"
     qt_right_semi_right_side "select t2.b from test1 t1 right semi join (select * from test2 where b=105)  t2 on t1.a=t2.a  group by t2.b,t2.a order by 1;"
     qt_right_anti_right_side "select t2.b from test1 t1 right anti join (select * from test2 where b=105)  t2 on t1.a=t2.a  group by t2.b,t2.a order by 1;"
+
+    //grouping
+    qt_grouping "select k, k3 from (select 1 as k, a k3, sum(b) as sum_k1 from  test1  group by cube(k,a)) t group by k,k3 order by 1,2"
 }
\ No newline at end of file

From ff61365049e9acbd209ae4c7e48603caff5949c8 Mon Sep 17 00:00:00 2001
From: qiye <luen@selectdb.com>
Date: Wed, 8 Jan 2025 10:06:11 +0800
Subject: [PATCH 035/140] [fix](ES catalog)Fix query long value exception with
 doc_value (#46554)

Issue Number: close #46553

Problem Summary:

When parsing number value from pure_doc_value or array, we should get
the first value, not the whole array.
---
 be/src/exec/es/es_scroll_parser.cpp           |   3 +-
 .../elasticsearch/scripts/data/data1_es6.json |   1 +
 .../elasticsearch/scripts/data/data2_es6.json |   1 +
 .../elasticsearch/scripts/data/data3_es6.json |   1 +
 .../elasticsearch/scripts/data/data4_es6.json |   1 +
 .../scripts/index/es6_test1.json              |   3 +
 .../scripts/index/es6_test2.json              |   3 +
 .../external_table_p0/es/test_es_query.out    | 312 +++++++++++++-----
 .../es/test_es_query_no_http_url.out          |   2 +-
 .../external_table_p0/es/test_es_query.groovy |  13 +
 10 files changed, 257 insertions(+), 83 deletions(-)

diff --git a/be/src/exec/es/es_scroll_parser.cpp b/be/src/exec/es/es_scroll_parser.cpp
index f745ac34e65a0c2..6067203f2ba7aa5 100644
--- a/be/src/exec/es/es_scroll_parser.cpp
+++ b/be/src/exec/es/es_scroll_parser.cpp
@@ -401,8 +401,7 @@ Status insert_int_value(const rapidjson::Value& col, PrimitiveType type,
     };
 
     if (pure_doc_value && col.IsArray() && !col.Empty()) {
-        if (col.IsNumber()) {
-            RETURN_ERROR_IF_COL_IS_NOT_NUMBER(col[0], type);
+        if (col[0].IsNumber()) {
             T value = (T)(sizeof(T) < 8 ? col[0].GetInt() : col[0].GetInt64());
             col_ptr->insert_data(const_cast<const char*>(reinterpret_cast<char*>(&value)), 0);
             return Status::OK();
diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data1_es6.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data1_es6.json
index 8a6b404f8e6d147..8fdec33697a5802 100755
--- a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data1_es6.json
+++ b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data1_es6.json
@@ -5,6 +5,7 @@
   "test4": "2022-08-08",
   "test5": 12345,
   "test6": "2022-08-08T12:10:10.151Z",
+  "test7": 123456789,
   "c_bool": [true, false, true, true],
   "c_byte": [1, -2, -3, 4],
   "c_short": [128, 129, -129, -130],
diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data2_es6.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data2_es6.json
index 29ab899e9bd2128..71e8bae2811bb8f 100755
--- a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data2_es6.json
+++ b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data2_es6.json
@@ -5,6 +5,7 @@
   "test4": "2022-08-08",
   "test5": 2222.2,
   "test6": "2022-08-08T12:10:10.151+08:00",
+  "test7": "123456789",
   "c_bool": [true, false, true, true],
   "c_byte": [1, -2, -3, 4],
   "c_short": [128, 129, -129, -130],
diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data3_es6.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data3_es6.json
index 9e9dc0d424b0e6a..98952f3a3ced938 100755
--- a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data3_es6.json
+++ b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data3_es6.json
@@ -5,6 +5,7 @@
   "test4": "2022-08-08",
   "test5": "3333.22",
   "test6": "2022-08-08T12:10:10.151",
+  "test7": 1660104.44,
   "c_bool": [true, false, true, true],
   "c_byte": [1, -2, -3, 4],
   "c_short": [128, 129, -129, -130],
diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data4_es6.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data4_es6.json
index b639879bab8b8dd..bb99590369a6086 100755
--- a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data4_es6.json
+++ b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data4_es6.json
@@ -5,6 +5,7 @@
   "test4": "2022-08-08",
   "test5": 3333.22,
   "test6": "2022-08-08T12:10:10.151",
+  "test7": "1660104.44",
   "c_bool": [true, false, true, true],
   "c_byte": [1, -2, -3, 4],
   "c_short": [128, 129, -129, -130],
diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_test1.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_test1.json
index cfe83d127f2e3de..7f81005deaf4a4e 100755
--- a/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_test1.json
+++ b/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_test1.json
@@ -31,6 +31,9 @@
         "test6": {
           "type": "date"
         },
+        "test7": {
+          "type": "long"
+        },
         "c_bool": {
           "type": "boolean"
         },
diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_test2.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_test2.json
index f25faa35fc3b5bc..1e79f38ec6ec207 100755
--- a/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_test2.json
+++ b/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_test2.json
@@ -33,6 +33,9 @@
         "test6": {
           "type": "date"
         },
+        "test7": {
+          "type": "long"
+        },
         "c_bool": {
           "type": "boolean"
         },
diff --git a/regression-test/data/external_table_p0/es/test_es_query.out b/regression-test/data/external_table_p0/es/test_es_query.out
index 312aef06652ce5b..70cc0f4b04a5cde 100644
--- a/regression-test/data/external_table_p0/es/test_es_query.out
+++ b/regression-test/data/external_table_p0/es/test_es_query.out
@@ -1,9 +1,9 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !sql01 --
-["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	\N	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
+["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	12345	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	\N	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
 
 -- !sql02 --
-["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	\N	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
+["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	12345	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	\N	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
 
 -- !sql03 --
 2022-08-08	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T04:10:10	2022-08-08T20:10:10
@@ -23,11 +23,11 @@ I'm not null or empty
 I'm not null or empty
 
 -- !sql07 --
-["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	\N	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
-["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string2	[1, 2, 3, 4]	2022-08-08	2022-08-09T12:10:10	text2	["2020-01-01", "2020-01-02"]	4.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-09T12:10:10	2022-08-09T12:10:10	2022-08-09T12:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]		[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
-["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string3	[1, 2, 3, 4]	2022-08-08	2022-08-10T12:10:10	text3_4*5	["2020-01-01", "2020-01-02"]	5.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-10T12:10:10	2022-08-10T12:10:10	2022-08-10T20:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	I'm not null or empty	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
-["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string4	[1, 2, 3, 4]	2022-08-08	2022-08-11T12:10:10	text3_4*5	["2020-01-01", "2020-01-02"]	6.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-11T12:10:10	2022-08-11T12:10:10	2022-08-11T11:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	\N	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
-["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string_ignore_above_10	[1, 2, 3, 4]	2022-08-08	2022-08-11T12:10:10	text_ignore_above_10	["2020-01-01", "2020-01-02"]	6.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-11T12:10:10	2022-08-11T12:10:10	2022-08-11T11:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	\N	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
+["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	12345	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	\N	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
+["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string2	[1, 2, 3, 4]	2022-08-08	2022-08-09T12:10:10	text2	["2020-01-01", "2020-01-02"]	4.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-09T12:10:10	2022-08-09T12:10:10	2022-08-09T12:10:10	2222	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]		[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
+["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string3	[1, 2, 3, 4]	2022-08-08	2022-08-10T12:10:10	text3_4*5	["2020-01-01", "2020-01-02"]	5.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-10T12:10:10	2022-08-10T12:10:10	2022-08-10T20:10:10	3333	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	I'm not null or empty	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
+["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string4	[1, 2, 3, 4]	2022-08-08	2022-08-11T12:10:10	text3_4*5	["2020-01-01", "2020-01-02"]	6.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-11T12:10:10	2022-08-11T12:10:10	2022-08-11T11:10:10	4444	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	\N	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
+["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string_ignore_above_10	[1, 2, 3, 4]	2022-08-08	2022-08-11T12:10:10	text_ignore_above_10	["2020-01-01", "2020-01-02"]	6.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-11T12:10:10	2022-08-11T12:10:10	2022-08-11T11:10:10	4444	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	\N	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
 
 -- !sql08 --
 [{"name":"Andy","age":18},{"name":"Tim","age":28}]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	"Andy"	"White"
@@ -57,11 +57,18 @@ text_ignore_above_10
 2022-08-11T12:10:10
 2022-08-11T12:10:10
 
+-- !sql12 --
+12345
+2222
+3333
+4444
+4444
+
 -- !sql20 --
-["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
+["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	12345	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
 
 -- !sql21 --
-["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
+["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	12345	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
 
 -- !sql22 --
 2022-08-08	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T04:10:10	2022-08-08T20:10:10
@@ -71,11 +78,11 @@ text_ignore_above_10
 2022-08-08	2022-08-11T12:10:10	2022-08-11T12:10:10	2022-08-11T12:10:10	2022-08-11T11:10:10
 
 -- !sql23 --
-["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
-["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string2	[1, 2, 3, 4]	2022-08-08	2022-08-09T12:10:10	text2	["2020-01-01", "2020-01-02"]	4.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-09T12:10:10	2022-08-09T12:10:10	2022-08-09T12:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
-["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string3	[1, 2, 3, 4]	2022-08-08	2022-08-10T12:10:10	text3_4*5	["2020-01-01", "2020-01-02"]	5.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-10T12:10:10	2022-08-10T12:10:10	2022-08-10T20:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
-["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string4	[1, 2, 3, 4]	2022-08-08	2022-08-11T12:10:10	text3_4*5	["2020-01-01", "2020-01-02"]	6.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-11T12:10:10	2022-08-11T12:10:10	2022-08-11T11:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
-["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string_ignore_above_10	[1, 2, 3, 4]	2022-08-08	2022-08-11T12:10:10	text_ignore_above_10	["2020-01-01", "2020-01-02"]	6.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-11T12:10:10	2022-08-11T12:10:10	2022-08-11T11:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
+["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	12345	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
+["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string2	[1, 2, 3, 4]	2022-08-08	2022-08-09T12:10:10	text2	["2020-01-01", "2020-01-02"]	4.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-09T12:10:10	2022-08-09T12:10:10	2022-08-09T12:10:10	2222	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
+["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string3	[1, 2, 3, 4]	2022-08-08	2022-08-10T12:10:10	text3_4*5	["2020-01-01", "2020-01-02"]	5.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-10T12:10:10	2022-08-10T12:10:10	2022-08-10T20:10:10	3333	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
+["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string4	[1, 2, 3, 4]	2022-08-08	2022-08-11T12:10:10	text3_4*5	["2020-01-01", "2020-01-02"]	6.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-11T12:10:10	2022-08-11T12:10:10	2022-08-11T11:10:10	4444	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
+["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string_ignore_above_10	[1, 2, 3, 4]	2022-08-08	2022-08-11T12:10:10	text_ignore_above_10	["2020-01-01", "2020-01-02"]	6.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-11T12:10:10	2022-08-11T12:10:10	2022-08-11T11:10:10	4444	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
 
 -- !sql24 --
 [{"name":"Andy","age":18},{"name":"Tim","age":28}]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	"Andy"	"White"
@@ -91,16 +98,23 @@ text_ignore_above_10
 2022-08-11T12:10:10
 2022-08-11T12:10:10
 
+-- !sql26 --
+12345
+2222
+3333
+4444
+4444
+
 -- !sql_5_02 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10	123456789
 
 -- !sql_5_03 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string3	text3_4*5	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10	123456789
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10	123456789
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string3	text3_4*5	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10	\N
 
 -- !sql_5_04 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10	123456789
 
 -- !sql_5_05 --
 true	1	128	32768	-1	0	1.0	1.0	1.0	1.0	2020-01-01	2020-01-01T12:00	a	d	192.168.0.1	{"name":"Andy","age":18}
@@ -114,7 +128,7 @@ true	1	128	32768	-1	0	1.0	1.0	1.0	1.0	2020-01-01	2020-01-01T12:00	a	d	192.168.0.
 true	1	128	32768	-1	0	1.0	1.0	1.0	1.0	2020-01-01	2020-01-01T12:00	a	d	192.168.0.1	{"name":"Andy","age":18}
 
 -- !sql_5_07 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10	123456789
 
 -- !sql_5_08 --
 [1, 0, 1, 1]	[1, -2, -3, 4]	[128, 129, -129, -130]	[32768, 32769, -32769, -32770]	[-1, 0, 1, 2]	[0, 1, 2, 3]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[1, 2, 3, 4]	[1, 2, 3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	["a", "b", "c"]	["d", "e", "f"]	["192.168.0.1", "127.0.0.1"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]
@@ -128,13 +142,13 @@ true	1	128	32768	-1	0	1.0	1.0	1.0	1.0	2020-01-01	2020-01-01T12:00	a	d	192.168.0.
 [1, 0, 1, 1]	[1, -2, -3, 4]	[128, 129, -129, -130]	[32768, 32769, -32769, -32770]	[-1, 0, 1, 2]	[0, 1, 2, 3]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[1, 2, 3, 4]	[1, 2, 3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	["a", "b", "c"]	["d", "e", "f"]	["192.168.0.1", "127.0.0.1"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]
 
 -- !sql_5_10 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10	123456789
 
 -- !sql_5_11 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10	123456789
 
 -- !sql_5_12 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string3	text3_4*5	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string3	text3_4*5	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10	\N
 
 -- !sql_5_13 --
 2022-08-08T20:10:10
@@ -159,10 +173,10 @@ I'm not null or empty
 I'm not null or empty
 
 -- !sql_5_19 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string3	text3_4*5	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string_ignore_above_10	text_ignore_above_10	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10	123456789
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10	123456789
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string3	text3_4*5	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10	\N
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string_ignore_above_10	text_ignore_above_10	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10	1660104
 
 -- !sql_5_20 --
 [{"name":"Andy","age":18},{"name":"Tim","age":28}]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	"Andy"	"White"
@@ -231,16 +245,30 @@ string2	text2
 string3	text3_4*5
 string_ignore_above_10	text_ignore_above_10
 
+-- !sql_5_31 --
+\N
+123456789
+123456789
+1660104
+
+-- !sql_5_32 --
+\N
+\N
+123456789
+123456789
+123456789
+123456789
+
 -- !sql_6_02 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10	123456789
 
 -- !sql_6_03 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string3	text3_4*5	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10	123456789
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10	123456789
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string3	text3_4*5	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10	1660104
 
 -- !sql_6_04 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10	123456789
 
 -- !sql_6_05 --
 true	1	128	32768	-1	0	1.0	1.0	1.0	1.0	2020-01-01	2020-01-01T12:00	a	d	192.168.0.1	{"name":"Andy","age":18}
@@ -254,7 +282,7 @@ true	1	128	32768	-1	0	1.0	1.0	1.0	1.0	2020-01-01	2020-01-01T12:00	a	d	192.168.0.
 true	1	128	32768	-1	0	1.0	1.0	1.0	1.0	2020-01-01	2020-01-01T12:00	a	d	192.168.0.1	{"name":"Andy","age":18}
 
 -- !sql_6_07 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10	123456789
 
 -- !sql_6_08 --
 [1, 0, 1, 1]	[1, -2, -3, 4]	[128, 129, -129, -130]	[32768, 32769, -32769, -32770]	[-1, 0, 1, 2]	[0, 1, 2, 3]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[1, 2, 3, 4]	[1, 2, 3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	["a", "b", "c"]	["d", "e", "f"]	["192.168.0.1", "127.0.0.1"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]
@@ -268,13 +296,13 @@ true	1	128	32768	-1	0	1.0	1.0	1.0	1.0	2020-01-01	2020-01-01T12:00	a	d	192.168.0.
 [1, 0, 1, 1]	[1, -2, -3, 4]	[128, 129, -129, -130]	[32768, 32769, -32769, -32770]	[-1, 0, 1, 2]	[0, 1, 2, 3]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[1, 2, 3, 4]	[1, 2, 3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	["a", "b", "c"]	["d", "e", "f"]	["192.168.0.1", "127.0.0.1"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]
 
 -- !sql_6_10 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10	123456789
 
 -- !sql_6_11 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10	123456789
 
 -- !sql_6_12 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string3	text3_4*5	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string3	text3_4*5	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10	1660104
 
 -- !sql_6_13 --
 2022-08-08T20:10:10
@@ -299,10 +327,10 @@ I'm not null or empty
 I'm not null or empty
 
 -- !sql_6_19 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string3	text3_4*5	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string_ignore_above_10	text_ignore_above_10	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10	123456789
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10	123456789
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string3	text3_4*5	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10	1660104
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string_ignore_above_10	text_ignore_above_10	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10	1660104
 
 -- !sql_6_20 --
 [{"name":"Andy","age":18},{"name":"Tim","age":28}]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	"Andy"	"White"
@@ -371,6 +399,20 @@ string2	text2
 string3	text3_4*5
 string_ignore_above_10	text_ignore_above_10
 
+-- !sql_6_31 --
+123456789
+123456789
+1660104
+1660104
+
+-- !sql_6_32 --
+123456789
+123456789
+123456789
+123456789
+1660104
+1660104
+
 -- !sql_7_02 --
 [1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	debug	\N	This string can be quite lengthy	string1	2022-08-08T20:10:10	text#1	3.14	2022-08-08T00:00	2022-08-08T12:10:10	1659931810000	2022-08-08T12:10:10	2022-08-08T20:10:10	12345
 
@@ -552,6 +594,23 @@ string3	text3_4*5
 string4	text3_4*5
 string_ignore_above_10	text_ignore_above_10
 
+-- !sql_7_37 --
+12345
+2222
+3333
+4444
+4444
+
+-- !sql_7_38 --
+12345
+12345
+2222
+2222
+3333
+3333
+4444
+4444
+
 -- !sql_7_50 --
 value1	value2
 
@@ -733,11 +792,28 @@ string3	text3_4*5
 string4	text3_4*5
 string_ignore_above_10	text_ignore_above_10
 
+-- !sql_8_35 --
+12345
+2222
+3333
+4444
+4444
+
+-- !sql_8_36 --
+12345
+12345
+2222
+2222
+3333
+3333
+4444
+4444
+
 -- !sql01 --
-["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	\N	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
+["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	12345	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	\N	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
 
 -- !sql02 --
-["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	\N	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
+["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	12345	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	\N	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
 
 -- !sql03 --
 2022-08-08	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T04:10:10	2022-08-08T20:10:10
@@ -757,11 +833,11 @@ I'm not null or empty
 I'm not null or empty
 
 -- !sql07 --
-["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	\N	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
-["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string2	[1, 2, 3, 4]	2022-08-08	2022-08-09T12:10:10	text2	["2020-01-01", "2020-01-02"]	4.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-09T12:10:10	2022-08-09T12:10:10	2022-08-09T12:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]		[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
-["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string3	[1, 2, 3, 4]	2022-08-08	2022-08-10T12:10:10	text3_4*5	["2020-01-01", "2020-01-02"]	5.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-10T12:10:10	2022-08-10T12:10:10	2022-08-10T20:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	I'm not null or empty	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
-["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string4	[1, 2, 3, 4]	2022-08-08	2022-08-11T12:10:10	text3_4*5	["2020-01-01", "2020-01-02"]	6.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-11T12:10:10	2022-08-11T12:10:10	2022-08-11T11:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	\N	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
-["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string_ignore_above_10	[1, 2, 3, 4]	2022-08-08	2022-08-11T12:10:10	text_ignore_above_10	["2020-01-01", "2020-01-02"]	6.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-11T12:10:10	2022-08-11T12:10:10	2022-08-11T11:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	\N	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
+["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	12345	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	\N	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
+["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string2	[1, 2, 3, 4]	2022-08-08	2022-08-09T12:10:10	text2	["2020-01-01", "2020-01-02"]	4.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-09T12:10:10	2022-08-09T12:10:10	2022-08-09T12:10:10	2222	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]		[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
+["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string3	[1, 2, 3, 4]	2022-08-08	2022-08-10T12:10:10	text3_4*5	["2020-01-01", "2020-01-02"]	5.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-10T12:10:10	2022-08-10T12:10:10	2022-08-10T20:10:10	3333	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	I'm not null or empty	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
+["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string4	[1, 2, 3, 4]	2022-08-08	2022-08-11T12:10:10	text3_4*5	["2020-01-01", "2020-01-02"]	6.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-11T12:10:10	2022-08-11T12:10:10	2022-08-11T11:10:10	4444	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	\N	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
+["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string_ignore_above_10	[1, 2, 3, 4]	2022-08-08	2022-08-11T12:10:10	text_ignore_above_10	["2020-01-01", "2020-01-02"]	6.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-11T12:10:10	2022-08-11T12:10:10	2022-08-11T11:10:10	4444	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	\N	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
 
 -- !sql08 --
 [{"name":"Andy","age":18},{"name":"Tim","age":28}]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	"Andy"	"White"
@@ -791,11 +867,18 @@ text_ignore_above_10
 2022-08-11T12:10:10
 2022-08-11T12:10:10
 
+-- !sql12 --
+12345
+2222
+3333
+4444
+4444
+
 -- !sql20 --
-["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
+["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	12345	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
 
 -- !sql21 --
-["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
+["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	12345	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
 
 -- !sql22 --
 2022-08-08	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T04:10:10	2022-08-08T20:10:10
@@ -805,11 +888,11 @@ text_ignore_above_10
 2022-08-08	2022-08-11T12:10:10	2022-08-11T12:10:10	2022-08-11T12:10:10	2022-08-11T11:10:10
 
 -- !sql23 --
-["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
-["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string2	[1, 2, 3, 4]	2022-08-08	2022-08-09T12:10:10	text2	["2020-01-01", "2020-01-02"]	4.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-09T12:10:10	2022-08-09T12:10:10	2022-08-09T12:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
-["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string3	[1, 2, 3, 4]	2022-08-08	2022-08-10T12:10:10	text3_4*5	["2020-01-01", "2020-01-02"]	5.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-10T12:10:10	2022-08-10T12:10:10	2022-08-10T20:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
-["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string4	[1, 2, 3, 4]	2022-08-08	2022-08-11T12:10:10	text3_4*5	["2020-01-01", "2020-01-02"]	6.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-11T12:10:10	2022-08-11T12:10:10	2022-08-11T11:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
-["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string_ignore_above_10	[1, 2, 3, 4]	2022-08-08	2022-08-11T12:10:10	text_ignore_above_10	["2020-01-01", "2020-01-02"]	6.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-11T12:10:10	2022-08-11T12:10:10	2022-08-11T11:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
+["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	12345	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
+["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string2	[1, 2, 3, 4]	2022-08-08	2022-08-09T12:10:10	text2	["2020-01-01", "2020-01-02"]	4.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-09T12:10:10	2022-08-09T12:10:10	2022-08-09T12:10:10	2222	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
+["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string3	[1, 2, 3, 4]	2022-08-08	2022-08-10T12:10:10	text3_4*5	["2020-01-01", "2020-01-02"]	5.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-10T12:10:10	2022-08-10T12:10:10	2022-08-10T20:10:10	3333	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
+["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string4	[1, 2, 3, 4]	2022-08-08	2022-08-11T12:10:10	text3_4*5	["2020-01-01", "2020-01-02"]	6.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-11T12:10:10	2022-08-11T12:10:10	2022-08-11T11:10:10	4444	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
+["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string_ignore_above_10	[1, 2, 3, 4]	2022-08-08	2022-08-11T12:10:10	text_ignore_above_10	["2020-01-01", "2020-01-02"]	6.0	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	2022-08-11T12:10:10	2022-08-11T12:10:10	2022-08-11T11:10:10	4444	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]
 
 -- !sql24 --
 [{"name":"Andy","age":18},{"name":"Tim","age":28}]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	"Andy"	"White"
@@ -825,16 +908,23 @@ text_ignore_above_10
 2022-08-11T12:10:10
 2022-08-11T12:10:10
 
+-- !sql26 --
+12345
+2222
+3333
+4444
+4444
+
 -- !sql_5_02 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10	123456789
 
 -- !sql_5_03 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string3	text3_4*5	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10	123456789
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10	123456789
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string3	text3_4*5	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10	\N
 
 -- !sql_5_04 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10	123456789
 
 -- !sql_5_05 --
 true	1	128	32768	-1	0	1.0	1.0	1.0	1.0	2020-01-01	2020-01-01T12:00	a	d	192.168.0.1	{"name":"Andy","age":18}
@@ -848,7 +938,7 @@ true	1	128	32768	-1	0	1.0	1.0	1.0	1.0	2020-01-01	2020-01-01T12:00	a	d	192.168.0.
 true	1	128	32768	-1	0	1.0	1.0	1.0	1.0	2020-01-01	2020-01-01T12:00	a	d	192.168.0.1	{"name":"Andy","age":18}
 
 -- !sql_5_07 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10	123456789
 
 -- !sql_5_08 --
 [1, 0, 1, 1]	[1, -2, -3, 4]	[128, 129, -129, -130]	[32768, 32769, -32769, -32770]	[-1, 0, 1, 2]	[0, 1, 2, 3]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[1, 2, 3, 4]	[1, 2, 3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	["a", "b", "c"]	["d", "e", "f"]	["192.168.0.1", "127.0.0.1"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]
@@ -862,13 +952,13 @@ true	1	128	32768	-1	0	1.0	1.0	1.0	1.0	2020-01-01	2020-01-01T12:00	a	d	192.168.0.
 [1, 0, 1, 1]	[1, -2, -3, 4]	[128, 129, -129, -130]	[32768, 32769, -32769, -32770]	[-1, 0, 1, 2]	[0, 1, 2, 3]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[1, 2, 3, 4]	[1, 2, 3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	["a", "b", "c"]	["d", "e", "f"]	["192.168.0.1", "127.0.0.1"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]
 
 -- !sql_5_10 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10	123456789
 
 -- !sql_5_11 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10	123456789
 
 -- !sql_5_12 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string3	text3_4*5	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string3	text3_4*5	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10	\N
 
 -- !sql_5_13 --
 2022-08-08T20:10:10
@@ -893,10 +983,10 @@ I'm not null or empty
 I'm not null or empty
 
 -- !sql_5_19 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string3	text3_4*5	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string_ignore_above_10	text_ignore_above_10	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10	123456789
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10	123456789
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string3	text3_4*5	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10	\N
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string_ignore_above_10	text_ignore_above_10	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10	1660104
 
 -- !sql_5_20 --
 [{"name":"Andy","age":18},{"name":"Tim","age":28}]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	"Andy"	"White"
@@ -965,16 +1055,30 @@ string2	text2
 string3	text3_4*5
 string_ignore_above_10	text_ignore_above_10
 
+-- !sql_5_31 --
+\N
+123456789
+123456789
+1660104
+
+-- !sql_5_32 --
+\N
+\N
+123456789
+123456789
+123456789
+123456789
+
 -- !sql_6_02 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10	123456789
 
 -- !sql_6_03 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string3	text3_4*5	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10	123456789
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10	123456789
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string3	text3_4*5	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10	1660104
 
 -- !sql_6_04 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10	123456789
 
 -- !sql_6_05 --
 true	1	128	32768	-1	0	1.0	1.0	1.0	1.0	2020-01-01	2020-01-01T12:00	a	d	192.168.0.1	{"name":"Andy","age":18}
@@ -988,7 +1092,7 @@ true	1	128	32768	-1	0	1.0	1.0	1.0	1.0	2020-01-01	2020-01-01T12:00	a	d	192.168.0.
 true	1	128	32768	-1	0	1.0	1.0	1.0	1.0	2020-01-01	2020-01-01T12:00	a	d	192.168.0.1	{"name":"Andy","age":18}
 
 -- !sql_6_07 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10	123456789
 
 -- !sql_6_08 --
 [1, 0, 1, 1]	[1, -2, -3, 4]	[128, 129, -129, -130]	[32768, 32769, -32769, -32770]	[-1, 0, 1, 2]	[0, 1, 2, 3]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[1, 2, 3, 4]	[1, 2, 3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	["a", "b", "c"]	["d", "e", "f"]	["192.168.0.1", "127.0.0.1"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]
@@ -1002,13 +1106,13 @@ true	1	128	32768	-1	0	1.0	1.0	1.0	1.0	2020-01-01	2020-01-01T12:00	a	d	192.168.0.
 [1, 0, 1, 1]	[1, -2, -3, 4]	[128, 129, -129, -130]	[32768, 32769, -32769, -32770]	[-1, 0, 1, 2]	[0, 1, 2, 3]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[1, 2, 3, 4]	[1, 2, 3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	["a", "b", "c"]	["d", "e", "f"]	["192.168.0.1", "127.0.0.1"]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]
 
 -- !sql_6_10 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10	123456789
 
 -- !sql_6_11 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10	123456789
 
 -- !sql_6_12 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string3	text3_4*5	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string3	text3_4*5	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10	1660104
 
 -- !sql_6_13 --
 2022-08-08T20:10:10
@@ -1033,10 +1137,10 @@ I'm not null or empty
 I'm not null or empty
 
 -- !sql_6_19 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string3	text3_4*5	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string_ignore_above_10	text_ignore_above_10	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10	123456789
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10	123456789
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string3	text3_4*5	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10	1660104
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	I'm not null or empty	string_ignore_above_10	text_ignore_above_10	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10	1660104
 
 -- !sql_6_20 --
 [{"name":"Andy","age":18},{"name":"Tim","age":28}]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	"Andy"	"White"
@@ -1105,6 +1209,20 @@ string2	text2
 string3	text3_4*5
 string_ignore_above_10	text_ignore_above_10
 
+-- !sql_6_31 --
+123456789
+123456789
+1660104
+1660104
+
+-- !sql_6_32 --
+123456789
+123456789
+123456789
+123456789
+1660104
+1660104
+
 -- !sql_7_02 --
 [1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	debug	\N	This string can be quite lengthy	string1	2022-08-08T20:10:10	text#1	3.14	2022-08-08T00:00	2022-08-08T12:10:10	1659931810000	2022-08-08T12:10:10	2022-08-08T20:10:10	12345
 
@@ -1286,6 +1404,23 @@ string3	text3_4*5
 string4	text3_4*5
 string_ignore_above_10	text_ignore_above_10
 
+-- !sql_7_37 --
+12345
+2222
+3333
+4444
+4444
+
+-- !sql_7_38 --
+12345
+12345
+2222
+2222
+3333
+3333
+4444
+4444
+
 -- !sql_7_50 --
 value1	value2
 
@@ -1467,3 +1602,20 @@ string3	text3_4*5
 string4	text3_4*5
 string_ignore_above_10	text_ignore_above_10
 
+-- !sql_8_35 --
+12345
+2222
+3333
+4444
+4444
+
+-- !sql_8_36 --
+12345
+12345
+2222
+2222
+3333
+3333
+4444
+4444
+
diff --git a/regression-test/data/external_table_p0/es/test_es_query_no_http_url.out b/regression-test/data/external_table_p0/es/test_es_query_no_http_url.out
index 60454994e1b5029..a1922f1f55d25d4 100644
--- a/regression-test/data/external_table_p0/es/test_es_query_no_http_url.out
+++ b/regression-test/data/external_table_p0/es/test_es_query_no_http_url.out
@@ -6,7 +6,7 @@
 ["2020-01-01", "2020-01-02"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]
 
 -- !sql61 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10	123456789
 
 -- !sql71 --
 [1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	[{"name":"Andy","age":18},{"name":"Tim","age":28}]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	[{"last":"Smith","first":"John"},{"last":"White","first":"Alice"}]	debug	\N	This string can be quite lengthy	string1	2022-08-08T20:10:10	text#1	3.14	2022-08-08T00:00	2022-08-08T12:10:10	1659931810000	2022-08-08T12:10:10	2022-08-08T20:10:10	12345
diff --git a/regression-test/suites/external_table_p0/es/test_es_query.groovy b/regression-test/suites/external_table_p0/es/test_es_query.groovy
index a73df1ed5e10967..1645fa6af51bfee 100644
--- a/regression-test/suites/external_table_p0/es/test_es_query.groovy
+++ b/regression-test/suites/external_table_p0/es/test_es_query.groovy
@@ -110,6 +110,7 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") {
                 `test6` datetime NULL,
                 `test7` datetime NULL,
                 `test8` datetime NULL,
+                `test9` bigint NULL,
                 `c_byte` array<tinyint(4)> NULL,
                 `c_bool` array<boolean> NULL,
                 `c_integer` array<int(11)> NULL,
@@ -149,6 +150,7 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") {
                 `test6` datetimev2 NULL,
                 `test7` datetimev2 NULL,
                 `test8` datetimev2 NULL,
+                `test9` bigint NULL,
                 `c_byte` array<tinyint(4)> NULL,
                 `c_bool` array<boolean> NULL,
                 `c_integer` array<int(11)> NULL,
@@ -178,6 +180,7 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") {
             order_qt_sql09 """select test1 from test_v1;"""
             order_qt_sql10 """select test2 from test_v1;"""
             order_qt_sql11 """select test6 from test_v1;"""
+            order_qt_sql12 """select test9 from test_v1;"""
             
             order_qt_sql20 """select * from test_v2 where test2='text#1'"""
             order_qt_sql21 """select * from test_v2 where esquery(test2, '{"match":{"test2":"text#1"}}')"""
@@ -185,6 +188,7 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") {
             order_qt_sql23 """select * from test_v2 where esquery(c_long, '{"term":{"c_long":"-1"}}');"""
             order_qt_sql24 """select c_person, c_user, json_extract(c_person, '\$.[0].name'), json_extract(c_user, '\$.[1].last') from test_v2;"""
             order_qt_sql25 """select test6 from test_v2;"""
+            order_qt_sql26 """select test9 from test_v2;"""
 
             sql """switch test_es_query_es5"""
             order_qt_sql_5_02 """select * from test1 where test2='text#1'"""
@@ -216,6 +220,8 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") {
             order_qt_sql_5_28 """select * from test3_20231005;"""
             order_qt_sql_5_29 """select test1, test2 from test1 where test1 like 'string%';"""
             order_qt_sql_5_30 """select test1, test2 from test1 where test2 like 'text%';"""
+            order_qt_sql_5_31 """select test7 from test1;"""
+            order_qt_sql_5_32 """select test7 from test2;"""
 
             sql """switch test_es_query_es6"""
             // order_qt_sql_6_01 """show tables"""
@@ -248,6 +254,9 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") {
             order_qt_sql_6_28 """select * from test3_20231005;"""
             order_qt_sql_6_29 """select test1, test2 from test1 where test1 like 'string%';"""
             order_qt_sql_6_30 """select test1, test2 from test1 where test2 like 'text%';"""
+            order_qt_sql_6_31 """select test7 from test1;"""
+            order_qt_sql_6_32 """select test7 from test2;"""
+
 
             List<List<String>> tables6N = sql """show tables"""
             boolean notContainHide = true
@@ -305,6 +314,8 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") {
             order_qt_sql_7_34 """select * from composite_type_array order by name;"""
             order_qt_sql_7_35 """select test1, test2 from test1 where test1 like 'string%';"""
             order_qt_sql_7_36 """select test1, test2 from test1 where test2 like 'text%';"""
+            order_qt_sql_7_37 """select test9 from test1;"""
+            order_qt_sql_7_38 """select test9 from test2;"""
 
             List<List<String>> tables7N = sql """show tables"""
             boolean notContainHide7 = true
@@ -362,6 +373,8 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") {
             order_qt_sql_8_32 """select * from composite_type_array order by name;"""
             order_qt_sql_8_33 """select test1, test2 from test1 where test1 like 'string%';"""
             order_qt_sql_8_34 """select test1, test2 from test1 where test2 like 'text%';"""
+            order_qt_sql_8_35 """select test9 from test1;"""
+            order_qt_sql_8_36 """select test9 from test2;"""
 
         }
 

From c3ed2d310ed1cc9447e078eac9bc50fe96d14460 Mon Sep 17 00:00:00 2001
From: qiye <luen@selectdb.com>
Date: Wed, 8 Jan 2025 10:07:04 +0800
Subject: [PATCH 036/140] [fix](test)Fix unstable index_change_with_cumu case
 (#46527)

---
 ...x_change_with_cumulative_compaction.groovy | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/regression-test/suites/inverted_index_p0/index_change/test_index_change_with_cumulative_compaction.groovy b/regression-test/suites/inverted_index_p0/index_change/test_index_change_with_cumulative_compaction.groovy
index 27789533420d2fb..ce12d1ede0cc98a 100644
--- a/regression-test/suites/inverted_index_p0/index_change/test_index_change_with_cumulative_compaction.groovy
+++ b/regression-test/suites/inverted_index_p0/index_change/test_index_change_with_cumulative_compaction.groovy
@@ -40,6 +40,23 @@ suite("test_index_change_with_cumulative_compaction", "nonConcurrent") {
         assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout")
     }
 
+    def trigger_compaction_with_retry = {table_name, compaction_type = "cumulative", max_retries = 10, delay_ms = 2000 ->
+        def retry_count = 0
+        while (true) {
+            try {
+                trigger_and_wait_compaction(table_name, compaction_type)
+                return // Success
+            } catch (Exception e) {
+                retry_count++
+                if (retry_count >= max_retries) {
+                    throw new Exception("Failed to complete ${compaction_type} compaction after ${max_retries} attempts", e)
+                }
+                logger.warn("Compaction attempt ${retry_count} failed: ${e.getMessage()}")
+                Thread.sleep(delay_ms)
+            }
+        }
+    }
+
     try {
         //BackendId,Cluster,IP,HeartbeatPort,BePort,HttpPort,BrpcPort,LastStartTime,LastHeartbeat,Alive,SystemDecommissioned,ClusterDecommissioned,TabletNum,DataUsedCapacity,AvailCapacity,TotalCapacity,UsedPct,MaxDiskUsedPct,Tag,ErrMsg,Version,Status
         String[][] backends = sql """ show backends; """
@@ -150,7 +167,7 @@ suite("test_index_change_with_cumulative_compaction", "nonConcurrent") {
         }
 
         // trigger compactions for all tablets in ${tableName}
-        trigger_and_wait_compaction(tableName, "cumulative")
+        trigger_compaction_with_retry(tableName, "cumulative")
 
         int rowCount = 0
         for (def tablet in tablets) {

From 811f93682d58eae76f87c411fcb2a572f03b2c92 Mon Sep 17 00:00:00 2001
From: minghong <zhouminghong@selectdb.com>
Date: Wed, 8 Jan 2025 10:25:44 +0800
Subject: [PATCH 037/140] [fix](nereids) topN filter: use ObjectId as map key
 instead of Topn node  (#46551)

### What problem does this PR solve?
Plan node is not good to be hash map key, because two plan nodes in
different tree level may be regarded as "equal". for example, in
following tree, topn1.equals(topn2) may be true.
 Topn filter generator should distinguish them, and hence topn
node is not suitable to be used as hash map key.

topn1
    -->some node
        -->topn2
             -->other node

Related PR: #31485
---
 .../nereids/processor/post/TopnFilterContext.java | 15 ++++++++-------
 .../doris/nereids/trees/plans/algebra/TopN.java   |  4 ++++
 .../logical/LogicalDeferMaterializeTopN.java      |  6 ++++++
 .../nereids/trees/plans/logical/LogicalTopN.java  |  6 ++++++
 .../physical/PhysicalDeferMaterializeTopN.java    |  6 ++++++
 .../trees/plans/physical/PhysicalTopN.java        |  5 +++++
 6 files changed, 35 insertions(+), 7 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopnFilterContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopnFilterContext.java
index fceec21ee7e0463..5df829e57966ec0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopnFilterContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopnFilterContext.java
@@ -21,6 +21,7 @@
 import org.apache.doris.nereids.glue.translator.ExpressionTranslator;
 import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
 import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.plans.ObjectId;
 import org.apache.doris.nereids.trees.plans.algebra.TopN;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation;
 import org.apache.doris.nereids.trees.plans.physical.TopnFilter;
@@ -38,22 +39,22 @@
  * topN runtime filter context
  */
 public class TopnFilterContext {
-    private final Map<TopN, TopnFilter> filters = Maps.newHashMap();
+    private final Map<ObjectId, TopnFilter> filters = Maps.newHashMap();
 
     /**
      * add topN filter
      */
     public void addTopnFilter(TopN topn, PhysicalRelation scan, Expression expr) {
-        TopnFilter filter = filters.get(topn);
+        TopnFilter filter = filters.get(topn.getObjectId());
         if (filter == null) {
-            filters.put(topn, new TopnFilter(topn, scan, expr));
+            filters.put(topn.getObjectId(), new TopnFilter(topn, scan, expr));
         } else {
             filter.addTarget(scan, expr);
         }
     }
 
     public boolean isTopnFilterSource(TopN topn) {
-        return filters.containsKey(topn);
+        return filters.containsKey(topn.getObjectId());
     }
 
     public List<TopnFilter> getTopnFilters() {
@@ -77,7 +78,7 @@ public void translateTarget(PhysicalRelation relation, ScanNode legacyScan,
      * translate topn-filter
      */
     public void translateSource(TopN topn, SortNode sortNode) {
-        TopnFilter filter = filters.get(topn);
+        TopnFilter filter = filters.get(topn.getObjectId());
         if (filter == null) {
             return;
         }
@@ -97,8 +98,8 @@ public String toString() {
         String indent = "   ";
         String arrow = " -> ";
         builder.append("filters:\n");
-        for (TopN topn : filters.keySet()) {
-            builder.append(indent).append(arrow).append(filters.get(topn)).append("\n");
+        for (ObjectId topnId : filters.keySet()) {
+            builder.append(indent).append(arrow).append(filters.get(topnId)).append("\n");
         }
         return builder.toString();
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/TopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/TopN.java
index c214dffbbf0a8f3..ed4b71ee91aa918 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/TopN.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/TopN.java
@@ -17,6 +17,8 @@
 
 package org.apache.doris.nereids.trees.plans.algebra;
 
+import org.apache.doris.nereids.trees.plans.ObjectId;
+
 /**
  * Common interface for logical/physical TopN.
  */
@@ -25,4 +27,6 @@ public interface TopN extends Sort {
     long getOffset();
 
     long getLimit();
+
+    ObjectId getObjectId();
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java
index 9a0d03b52a1d853..5a791def8ee6f6d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java
@@ -25,6 +25,7 @@
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.plans.ObjectId;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.PlanType;
 import org.apache.doris.nereids.trees.plans.algebra.TopN;
@@ -198,4 +199,9 @@ public void computeFd(DataTrait.Builder builder) {
     public void computeEqualSet(DataTrait.Builder builder) {
         builder.addEqualSet(child().getLogicalProperties().getTrait());
     }
+
+    @Override
+    public ObjectId getObjectId() {
+        return id;
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java
index 9c7cfacad8dd33a..6d6aa2b8131bfd2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java
@@ -23,6 +23,7 @@
 import org.apache.doris.nereids.properties.OrderKey;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.plans.ObjectId;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.PlanType;
 import org.apache.doris.nereids.trees.plans.algebra.TopN;
@@ -186,4 +187,9 @@ public void computeEqualSet(DataTrait.Builder builder) {
     public void computeFd(DataTrait.Builder builder) {
         builder.addFuncDepsDG(child().getLogicalProperties().getTrait());
     }
+
+    @Override
+    public ObjectId getObjectId() {
+        return id;
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDeferMaterializeTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDeferMaterializeTopN.java
index f5db3ff42f8f1a0..74088498d723582 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDeferMaterializeTopN.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDeferMaterializeTopN.java
@@ -23,6 +23,7 @@
 import org.apache.doris.nereids.trees.expressions.ExprId;
 import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.plans.ObjectId;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.algebra.TopN;
 import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
@@ -175,4 +176,9 @@ public String toString() {
                 "columnIdSlot", columnIdSlot
         );
     }
+
+    @Override
+    public ObjectId getObjectId() {
+        return id;
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTopN.java
index c387a58dd0c993f..5f59f8a7083e1f0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTopN.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTopN.java
@@ -22,6 +22,7 @@
 import org.apache.doris.nereids.properties.OrderKey;
 import org.apache.doris.nereids.properties.PhysicalProperties;
 import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.plans.ObjectId;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.PlanType;
 import org.apache.doris.nereids.trees.plans.SortPhase;
@@ -162,4 +163,8 @@ public PhysicalTopN<Plan> resetLogicalProperties() {
                 null, physicalProperties, statistics, child());
     }
 
+    @Override
+    public ObjectId getObjectId() {
+        return id;
+    }
 }

From b26aabee2615f4db0b2571204ea0aa822f3a3d39 Mon Sep 17 00:00:00 2001
From: Lijia Liu <liutang123@yeah.net>
Date: Wed, 8 Jan 2025 10:26:33 +0800
Subject: [PATCH 038/140] [fix](pipelineX) fix DCHECK error when reuse
 _join_block in nested loop join (#46493)

Related Issue: #46498

Co-authored-by: liutang123 <liulijia@gmail.com>
---
 be/src/pipeline/exec/nested_loop_join_probe_operator.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/be/src/pipeline/exec/nested_loop_join_probe_operator.cpp b/be/src/pipeline/exec/nested_loop_join_probe_operator.cpp
index c9642bb9b3bd20f..e2c6b7d84f02f85 100644
--- a/be/src/pipeline/exec/nested_loop_join_probe_operator.cpp
+++ b/be/src/pipeline/exec/nested_loop_join_probe_operator.cpp
@@ -526,8 +526,7 @@ Status NestedLoopJoinProbeOperatorX::pull(RuntimeState* state, vectorized::Block
                         local_state._conjuncts, &local_state._join_block,
                         local_state._join_block.columns()));
             }
-            RETURN_IF_ERROR(
-                    local_state._build_output_block(&local_state._join_block, block, false));
+            RETURN_IF_ERROR(local_state._build_output_block(&local_state._join_block, block));
             local_state._reset_tuple_is_null_column();
         }
         local_state._join_block.clear_column_data(join_block_column_size);

From b0da4221d14d7a76ba8f957df9f864d44010818a Mon Sep 17 00:00:00 2001
From: Pxl <xl@selectdb.com>
Date: Wed, 8 Jan 2025 11:51:50 +0800
Subject: [PATCH 039/140] [Bug](join) fix columnstr64's offset overflow on
 serialize_value_into_arena (#46461)

### What problem does this PR solve?
```sql
mysql [test]>select /*+ LEADING(a,b) */ count(*) from d_table as a, d_table2 as b where a.k4=b.k4 and a.k1=b.k1;

+----------+
| count(*) |
+----------+
| 50000000 |
+----------+
1 row in set (4.87 sec)

mysql [test]>select /*+ LEADING(b,a) */ count(*) from d_table as a, d_table2 as b where a.k4=b.k4 and a.k1=b.k1;

+----------+
| count(*) |
+----------+
| 42949673 |
+----------+
1 row in set (21.32 sec)
```
### Release note

None

### Check List (For Author)

- Test <!-- At least one of them must be included. -->
    - [x] Regression test
    - [ ] Unit Test
    - [ ] Manual test (add detailed scripts or steps below)
    - [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
        - [ ] Previous test can cover this change.
        - [ ] No code files have been changed.
        - [ ] Other reason <!-- Add your reason?  -->

- Behavior changed:
    - [x] No.
    - [ ] Yes. <!-- Explain the behavior change -->

- Does this need documentation?
    - [x] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->

### Check List (For Reviewer who merge this PR)

- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
 be/src/vec/columns/column_string.cpp          | 22 ++++---
 be/src/vec/columns/column_string.h            | 10 +++-
 .../str64_serialize/str64_serialize.out       |  7 +++
 .../str64_serialize/str64_serialize.groovy    | 57 +++++++++++++++++++
 4 files changed, 81 insertions(+), 15 deletions(-)
 create mode 100644 regression-test/data/query_p1/str64_serialize/str64_serialize.out
 create mode 100644 regression-test/suites/query_p1/str64_serialize/str64_serialize.groovy

diff --git a/be/src/vec/columns/column_string.cpp b/be/src/vec/columns/column_string.cpp
index db0088e67c27b60..7bfae7c7a4946df 100644
--- a/be/src/vec/columns/column_string.cpp
+++ b/be/src/vec/columns/column_string.cpp
@@ -398,9 +398,8 @@ ColumnPtr ColumnStr<T>::permute(const IColumn::Permutation& perm, size_t limit)
 template <typename T>
 StringRef ColumnStr<T>::serialize_value_into_arena(size_t n, Arena& arena,
                                                    char const*& begin) const {
-    // Use uint32 instead of size_t to reduce agg key's length.
-    auto string_size(static_cast<uint32_t>(size_at(n)));
-    auto offset(static_cast<uint32_t>(offset_at(n)));
+    auto string_size(size_at(n));
+    auto offset(offset_at(n));
 
     StringRef res;
     res.size = sizeof(string_size) + string_size;
@@ -414,7 +413,7 @@ StringRef ColumnStr<T>::serialize_value_into_arena(size_t n, Arena& arena,
 
 template <typename T>
 const char* ColumnStr<T>::deserialize_and_insert_from_arena(const char* pos) {
-    const uint32_t string_size = unaligned_load<uint32_t>(pos);
+    const auto string_size = unaligned_load<uint32_t>(pos);
     pos += sizeof(string_size);
 
     const size_t old_size = chars.size();
@@ -432,7 +431,7 @@ size_t ColumnStr<T>::get_max_row_byte_size() const {
     T max_size = 0;
     size_t num_rows = offsets.size();
     for (size_t i = 0; i < num_rows; ++i) {
-        max_size = std::max(max_size, size_at(i));
+        max_size = std::max(max_size, T(size_at(i)));
     }
 
     return max_size + sizeof(uint32_t);
@@ -442,9 +441,8 @@ template <typename T>
 void ColumnStr<T>::serialize_vec(std::vector<StringRef>& keys, size_t num_rows,
                                  size_t max_row_byte_size) const {
     for (size_t i = 0; i < num_rows; ++i) {
-        // Use uint32 instead of size_t to reduce agg key's length.
-        auto offset(static_cast<uint32_t>(offset_at(i)));
-        auto string_size(static_cast<uint32_t>(size_at(i)));
+        auto offset(offset_at(i));
+        auto string_size(size_at(i));
 
         auto* ptr = const_cast<char*>(keys[i].data + keys[i].size);
         memcpy_fixed<uint32_t>(ptr, (char*)&string_size);
@@ -470,7 +468,7 @@ void ColumnStr<T>::serialize_vec_with_null_map(std::vector<StringRef>& keys, siz
                 auto offset(offset_at(i));
                 auto string_size(size_at(i));
 
-                memcpy_fixed<UInt32>(dest + 1, (char*)&string_size);
+                memcpy_fixed<uint32_t>(dest + 1, (char*)&string_size);
                 memcpy(dest + 1 + sizeof(string_size), &chars[offset], string_size);
                 keys[i].size += sizeof(string_size) + string_size + sizeof(UInt8);
             } else {
@@ -487,7 +485,7 @@ void ColumnStr<T>::serialize_vec_with_null_map(std::vector<StringRef>& keys, siz
             auto offset(offset_at(i));
             auto string_size(size_at(i));
 
-            memcpy_fixed<UInt32>(dest + 1, (char*)&string_size);
+            memcpy_fixed<uint32_t>(dest + 1, (char*)&string_size);
             memcpy(dest + 1 + sizeof(string_size), &chars[offset], string_size);
             keys[i].size += sizeof(string_size) + string_size + sizeof(UInt8);
         }
@@ -497,7 +495,7 @@ void ColumnStr<T>::serialize_vec_with_null_map(std::vector<StringRef>& keys, siz
 template <typename T>
 void ColumnStr<T>::deserialize_vec(std::vector<StringRef>& keys, const size_t num_rows) {
     for (size_t i = 0; i != num_rows; ++i) {
-        auto original_ptr = keys[i].data;
+        const auto* original_ptr = keys[i].data;
         keys[i].data = deserialize_and_insert_from_arena(original_ptr);
         keys[i].size -= keys[i].data - original_ptr;
     }
@@ -508,7 +506,7 @@ void ColumnStr<T>::deserialize_vec_with_null_map(std::vector<StringRef>& keys,
                                                  const size_t num_rows, const uint8_t* null_map) {
     for (size_t i = 0; i != num_rows; ++i) {
         if (null_map[i] == 0) {
-            auto original_ptr = keys[i].data;
+            const auto* original_ptr = keys[i].data;
             keys[i].data = deserialize_and_insert_from_arena(original_ptr);
             keys[i].size -= keys[i].data - original_ptr;
         } else {
diff --git a/be/src/vec/columns/column_string.h b/be/src/vec/columns/column_string.h
index 1674fd90933dbe1..cbd6d3b78417d77 100644
--- a/be/src/vec/columns/column_string.h
+++ b/be/src/vec/columns/column_string.h
@@ -29,6 +29,7 @@
 #include <typeinfo>
 #include <vector>
 
+#include "common/cast_set.h"
 #include "common/compiler_util.h" // IWYU pragma: keep
 #include "common/exception.h"
 #include "common/status.h"
@@ -89,8 +90,11 @@ class ColumnStr final : public COWHelper<IColumn, ColumnStr<T>> {
     // Start position of i-th element.
     T ALWAYS_INLINE offset_at(ssize_t i) const { return offsets[i - 1]; }
 
-    /// Size of i-th element, including terminating zero.
-    T ALWAYS_INLINE size_at(ssize_t i) const { return offsets[i] - offsets[i - 1]; }
+    // Size of i-th element, including terminating zero.
+    // assume that the length of a single element is less than 32-bit
+    uint32_t ALWAYS_INLINE size_at(ssize_t i) const {
+        return uint32_t(offsets[i] - offsets[i - 1]);
+    }
 
     template <bool positive>
     struct less;
@@ -357,7 +361,7 @@ class ColumnStr final : public COWHelper<IColumn, ColumnStr<T>> {
 
         for (size_t i = start_index; i < start_index + num; i++) {
             int32_t codeword = data_array[i];
-            auto& src = dict[codeword];
+            const auto& src = dict[codeword];
             memcpy(chars.data() + old_size, src.data, src.size);
             old_size += src.size;
         }
diff --git a/regression-test/data/query_p1/str64_serialize/str64_serialize.out b/regression-test/data/query_p1/str64_serialize/str64_serialize.out
new file mode 100644
index 000000000000000..99c168b99d4c2e3
--- /dev/null
+++ b/regression-test/data/query_p1/str64_serialize/str64_serialize.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !test --
+50000000
+
+-- !test --
+50000000
+
diff --git a/regression-test/suites/query_p1/str64_serialize/str64_serialize.groovy b/regression-test/suites/query_p1/str64_serialize/str64_serialize.groovy
new file mode 100644
index 000000000000000..b0e3ffa99e937f0
--- /dev/null
+++ b/regression-test/suites/query_p1/str64_serialize/str64_serialize.groovy
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("str64_serialize") {
+    
+    sql """ DROP TABLE IF EXISTS d_table; """
+    sql """ DROP TABLE IF EXISTS d_table2; """
+
+
+    sql """
+            create table d_table (
+                k1 int null,
+                k2 int not null,
+                k3 bigint null,
+                k4 varchar(100) null
+            )
+            duplicate key (k1,k2,k3)
+            distributed BY hash(k1) buckets 3
+            properties("replication_num" = "1");
+        """
+    sql """
+            create table d_table2 (
+                k1 int null,
+                k2 int not null,
+                k3 bigint null,
+                k4 varchar(100) null
+            )
+            duplicate key (k1,k2,k3)
+            distributed BY hash(k1) buckets 3
+            properties("replication_num" = "1");
+        """
+
+    sql """insert into d_table select 1,1,1,'1234567890abcdefghigalsdhaluihdicandejionxaoxwdeuhwenudzmwoedxneiowdxiowedjxneiowdjixoneiiexdnuiexef' from (select 1 k1) as t lateral view explode_numbers(50000000) tmp1 as e1;
+"""
+
+    sql """insert into d_table2 select 1,1,1,'1234567890abcdefghigalsdhaluihdicandejionxaoxwdeuhwenudzmwoedxneiowdxiowedjxneiowdjixoneiiexdnuiexef';
+"""
+    sql "set parallel_pipeline_task_num=1;"
+
+    qt_test "select /*+ LEADING(a,b) */ count(*) from d_table as a, d_table2 as b where a.k4=b.k4 and a.k1=b.k1;"
+    qt_test "select /*+ LEADING(b,a) */ count(*) from d_table as a, d_table2 as b where a.k4=b.k4 and a.k1=b.k1;"
+}
+

From 514247b90b0bcb1fd8076a407b7872676c417dfc Mon Sep 17 00:00:00 2001
From: Pxl <xl@selectdb.com>
Date: Wed, 8 Jan 2025 11:52:04 +0800
Subject: [PATCH 040/140] [Enchancement](join) do not build non-output column
 on join probe (#46203)

### What problem does this PR solve?
1. do not build non-output column on join probe
2. remove some unused code on `JoinProbeLocalState<SharedStateArg,
Derived>::_build_output_block`
3. do not print stack on `PInternalService::cancel_plan_fragment`
Problem Summary:

### Release note

None

### Check List (For Author)

- Test <!-- At least one of them must be included. -->
    - [ ] Regression test
    - [ ] Unit Test
    - [ ] Manual test (add detailed scripts or steps below)
    - [x] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
        - [x] Previous test can cover this change.
        - [ ] No code files have been changed.
        - [ ] Other reason <!-- Add your reason?  -->

- Behavior changed:
    - [x] No.
    - [ ] Yes. <!-- Explain the behavior change -->

- Does this need documentation?
    - [x] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->

### Check List (For Reviewer who merge this PR)

- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
 .../pipeline/exec/hashjoin_probe_operator.h   |  3 +-
 .../exec/join/process_hash_table_probe_impl.h | 79 ++++++++-------
 be/src/pipeline/exec/join_probe_operator.cpp  | 99 +++----------------
 be/src/service/internal_service.cpp           |  2 +-
 be/src/vec/core/block.cpp                     | 35 ++++---
 be/src/vec/exprs/vexpr_context.cpp            | 10 +-
 6 files changed, 83 insertions(+), 145 deletions(-)

diff --git a/be/src/pipeline/exec/hashjoin_probe_operator.h b/be/src/pipeline/exec/hashjoin_probe_operator.h
index 55a8835f55b9cfb..533aaefa5beef9d 100644
--- a/be/src/pipeline/exec/hashjoin_probe_operator.h
+++ b/be/src/pipeline/exec/hashjoin_probe_operator.h
@@ -73,8 +73,7 @@ class HashJoinProbeLocalState final
         return _shared_state->build_block;
     }
     bool empty_right_table_shortcut() const {
-        // !Base::_projections.empty() means nereids planner
-        return _shared_state->empty_right_table_need_probe_dispose && !Base::_projections.empty();
+        return _shared_state->empty_right_table_need_probe_dispose;
     }
     std::string debug_string(int indentation_level) const override;
 
diff --git a/be/src/pipeline/exec/join/process_hash_table_probe_impl.h b/be/src/pipeline/exec/join/process_hash_table_probe_impl.h
index 8c5b051b684bc13..eb4a36d17cfd003 100644
--- a/be/src/pipeline/exec/join/process_hash_table_probe_impl.h
+++ b/be/src/pipeline/exec/join/process_hash_table_probe_impl.h
@@ -27,6 +27,7 @@
 #include "util/simd/bits.h"
 #include "vec/columns/column_filter_helper.h"
 #include "vec/columns/column_nullable.h"
+#include "vec/columns/columns_number.h"
 #include "vec/common/hash_table/join_hash_table.h"
 #include "vec/exprs/vexpr_context.h"
 
@@ -68,11 +69,6 @@ void ProcessHashTableProbe<JoinOpType>::build_side_output_column(
         vectorized::MutableColumns& mcol, const std::vector<bool>& output_slot_flags, int size,
         bool have_other_join_conjunct, bool is_mark_join) {
     SCOPED_TIMER(_build_side_output_timer);
-    constexpr auto is_semi_anti_join = JoinOpType == TJoinOp::RIGHT_ANTI_JOIN ||
-                                       JoinOpType == TJoinOp::RIGHT_SEMI_JOIN ||
-                                       JoinOpType == TJoinOp::LEFT_ANTI_JOIN ||
-                                       JoinOpType == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN ||
-                                       JoinOpType == TJoinOp::LEFT_SEMI_JOIN;
 
     constexpr auto probe_all =
             JoinOpType == TJoinOp::LEFT_OUTER_JOIN || JoinOpType == TJoinOp::FULL_OUTER_JOIN;
@@ -81,9 +77,9 @@ void ProcessHashTableProbe<JoinOpType>::build_side_output_column(
     bool build_index_has_zero =
             (JoinOpType != TJoinOp::INNER_JOIN && JoinOpType != TJoinOp::RIGHT_OUTER_JOIN) ||
             have_other_join_conjunct || is_mark_join;
-    bool need_output = (!is_semi_anti_join || have_other_join_conjunct ||
-                        (is_mark_join && !_parent->_mark_join_conjuncts.empty())) &&
-                       size;
+    if (!size) {
+        return;
+    }
     // Dispose right tuple is null flags columns
     if (probe_all && !have_other_join_conjunct) {
         _tuple_is_null_right_flags->resize(size);
@@ -91,43 +87,55 @@ void ProcessHashTableProbe<JoinOpType>::build_side_output_column(
         for (int i = 0; i < size; ++i) {
             null_data[i] = _build_indexs[i] == 0;
         }
-        if (need_output && _need_calculate_build_index_has_zero) {
+        if (_need_calculate_build_index_has_zero) {
             build_index_has_zero = simd::contain_byte(null_data, size, 1);
         }
     }
 
-    if (need_output) {
-        if (!build_index_has_zero && _build_column_has_null.empty()) {
-            _need_calculate_build_index_has_zero = false;
-            _build_column_has_null.resize(output_slot_flags.size());
-            for (int i = 0; i < _right_col_len; i++) {
-                const auto& column = *_build_block->safe_get_by_position(i).column;
-                _build_column_has_null[i] = false;
-                if (output_slot_flags[i] && column.is_nullable()) {
-                    const auto& nullable = assert_cast<const vectorized::ColumnNullable&>(column);
-                    _build_column_has_null[i] = !simd::contain_byte(
-                            nullable.get_null_map_data().data() + 1, nullable.size() - 1, 1);
-                    _need_calculate_build_index_has_zero |= _build_column_has_null[i];
-                }
+    if (!build_index_has_zero && _build_column_has_null.empty()) {
+        _need_calculate_build_index_has_zero = false;
+        _build_column_has_null.resize(output_slot_flags.size());
+        for (int i = 0; i < _right_col_len; i++) {
+            const auto& column = *_build_block->safe_get_by_position(i).column;
+            _build_column_has_null[i] = false;
+            if (output_slot_flags[i] && column.is_nullable()) {
+                const auto& nullable = assert_cast<const vectorized::ColumnNullable&>(column);
+                _build_column_has_null[i] = !simd::contain_byte(
+                        nullable.get_null_map_data().data() + 1, nullable.size() - 1, 1);
+                _need_calculate_build_index_has_zero |= _build_column_has_null[i];
             }
         }
+    }
 
-        for (int i = 0; i < _right_col_len; i++) {
-            const auto& column = *_build_block->safe_get_by_position(i).column;
-            if (output_slot_flags[i]) {
-                if (!build_index_has_zero && _build_column_has_null[i]) {
-                    assert_cast<vectorized::ColumnNullable*>(mcol[i + _right_col_idx].get())
-                            ->insert_indices_from_not_has_null(column, _build_indexs.data(),
-                                                               _build_indexs.data() + size);
-                } else {
-                    mcol[i + _right_col_idx]->insert_indices_from(column, _build_indexs.data(),
-                                                                  _build_indexs.data() + size);
-                }
+    for (size_t i = 0; i < _right_col_len && i + _right_col_idx < mcol.size(); i++) {
+        const auto& column = *_build_block->safe_get_by_position(i).column;
+        if (output_slot_flags[i]) {
+            if (!build_index_has_zero && _build_column_has_null[i]) {
+                assert_cast<vectorized::ColumnNullable*>(mcol[i + _right_col_idx].get())
+                        ->insert_indices_from_not_has_null(column, _build_indexs.data(),
+                                                           _build_indexs.data() + size);
             } else {
-                mcol[i + _right_col_idx]->insert_many_defaults(size);
+                mcol[i + _right_col_idx]->insert_indices_from(column, _build_indexs.data(),
+                                                              _build_indexs.data() + size);
             }
+        } else if (i + _right_col_idx != _parent->_mark_column_id) {
+            mcol[i + _right_col_idx]->insert_default();
+            mcol[i + _right_col_idx] =
+                    vectorized::ColumnConst::create(std::move(mcol[i + _right_col_idx]), size);
         }
     }
+    if (_parent->_mark_column_id != -1) {
+        // resize mark column and fill with true
+        auto& mark_column =
+                assert_cast<vectorized::ColumnNullable&>(*mcol[_parent->_mark_column_id]);
+        mark_column.resize(size);
+        auto* null_map = mark_column.get_null_map_column().get_data().data();
+        auto* data = assert_cast<vectorized::ColumnUInt8&>(mark_column.get_nested_column())
+                             .get_data()
+                             .data();
+        std::fill(null_map, null_map + size, 0);
+        std::fill(data, data + size, 1);
+    }
 }
 
 template <int JoinOpType>
@@ -146,7 +154,8 @@ void ProcessHashTableProbe<JoinOpType>::probe_side_output_column(
                                              _probe_indexs.data() + size);
             }
         } else {
-            mcol[i]->insert_many_defaults(size);
+            mcol[i]->insert_default();
+            mcol[i] = vectorized::ColumnConst::create(std::move(mcol[i]), size);
         }
     }
 
diff --git a/be/src/pipeline/exec/join_probe_operator.cpp b/be/src/pipeline/exec/join_probe_operator.cpp
index 9a50d76a48ce8c2..2ebe3a9d0557660 100644
--- a/be/src/pipeline/exec/join_probe_operator.cpp
+++ b/be/src/pipeline/exec/join_probe_operator.cpp
@@ -17,6 +17,8 @@
 
 #include "join_probe_operator.h"
 
+#include <memory>
+
 #include "pipeline/exec/hashjoin_probe_operator.h"
 #include "pipeline/exec/nested_loop_join_probe_operator.h"
 #include "pipeline/exec/operator.h"
@@ -87,83 +89,10 @@ void JoinProbeLocalState<SharedStateArg, Derived>::_construct_mutable_join_block
 template <typename SharedStateArg, typename Derived>
 Status JoinProbeLocalState<SharedStateArg, Derived>::_build_output_block(
         vectorized::Block* origin_block, vectorized::Block* output_block, bool keep_origin) {
-    auto& p = Base::_parent->template cast<typename Derived::Parent>();
-    if (!Base::_projections.empty()) {
-        // In previous versions, the join node had a separate set of project structures,
-        // and you could see a 'todo' in the Thrift definition.
-        //  Here, we have refactored it, but considering upgrade compatibility, we still need to retain the old code.
-        if (!output_block->mem_reuse()) {
-            output_block->swap(origin_block->clone_empty());
-        }
-        output_block->swap(*origin_block);
-        return Status::OK();
-    }
-    SCOPED_TIMER(_build_output_block_timer);
-    auto is_mem_reuse = output_block->mem_reuse();
-    vectorized::MutableBlock mutable_block =
-            is_mem_reuse ? vectorized::MutableBlock(output_block)
-                         : vectorized::MutableBlock(
-                                   vectorized::VectorizedUtils::create_empty_columnswithtypename(
-                                           p.row_desc()));
-    auto rows = origin_block->rows();
-    // TODO: After FE plan support same nullable of output expr and origin block and mutable column
-    // we should replace `insert_column_datas` by `insert_range_from`
-
-    auto insert_column_datas = [keep_origin](auto& to, vectorized::ColumnPtr& from, size_t rows) {
-        if (to->is_nullable() && !from->is_nullable()) {
-            if (keep_origin || !from->is_exclusive()) {
-                auto& null_column = reinterpret_cast<vectorized::ColumnNullable&>(*to);
-                null_column.get_nested_column().insert_range_from(*from, 0, rows);
-                null_column.get_null_map_column().get_data().resize_fill(rows, 0);
-            } else {
-                to = make_nullable(from, false)->assume_mutable();
-            }
-        } else {
-            if (keep_origin || !from->is_exclusive()) {
-                to->insert_range_from(*from, 0, rows);
-            } else {
-                to = from->assume_mutable();
-            }
-        }
-    };
-    if (rows != 0) {
-        auto& mutable_columns = mutable_block.mutable_columns();
-        if (_output_expr_ctxs.empty()) {
-            DCHECK(mutable_columns.size() == p.row_desc().num_materialized_slots())
-                    << mutable_columns.size() << " " << p.row_desc().num_materialized_slots();
-            for (int i = 0; i < mutable_columns.size(); ++i) {
-                insert_column_datas(mutable_columns[i], origin_block->get_by_position(i).column,
-                                    rows);
-            }
-        } else {
-            DCHECK(mutable_columns.size() == p.row_desc().num_materialized_slots())
-                    << mutable_columns.size() << " " << p.row_desc().num_materialized_slots();
-            SCOPED_TIMER(Base::_projection_timer);
-            for (int i = 0; i < mutable_columns.size(); ++i) {
-                auto result_column_id = -1;
-                RETURN_IF_ERROR(_output_expr_ctxs[i]->execute(origin_block, &result_column_id));
-                auto& origin_column = origin_block->get_by_position(result_column_id).column;
-
-                /// `convert_to_full_column_if_const` will create a pointer to the origin column if
-                /// the origin column is not ColumnConst/ColumnArray, this make the column be not
-                /// exclusive.
-                /// TODO: maybe need a method to check if a column need to be converted to full
-                /// column.
-                if (is_column_const(*origin_column) ||
-                    check_column<vectorized::ColumnArray>(origin_column.get())) {
-                    auto column_ptr = origin_column->convert_to_full_column_if_const();
-                    insert_column_datas(mutable_columns[i], column_ptr, rows);
-                } else {
-                    insert_column_datas(mutable_columns[i], origin_column, rows);
-                }
-            }
-        }
-
-        output_block->swap(mutable_block.to_block());
-
-        DCHECK(output_block->rows() == rows);
+    if (!output_block->mem_reuse()) {
+        output_block->swap(origin_block->clone_empty());
     }
-
+    output_block->swap(*origin_block);
     return Status::OK();
 }
 
@@ -221,20 +150,22 @@ JoinProbeOperatorX<LocalStateType>::JoinProbeOperatorX(ObjectPool* pool, const T
           ) {
     Base::_is_serial_operator = tnode.__isset.is_serial_operator && tnode.is_serial_operator;
     if (tnode.__isset.hash_join_node) {
-        _intermediate_row_desc.reset(new RowDescriptor(
+        _intermediate_row_desc = std::make_unique<RowDescriptor>(
                 descs, tnode.hash_join_node.vintermediate_tuple_id_list,
-                std::vector<bool>(tnode.hash_join_node.vintermediate_tuple_id_list.size())));
+                std::vector<bool>(tnode.hash_join_node.vintermediate_tuple_id_list.size()));
         if (!Base::_output_row_descriptor) {
-            _output_row_desc.reset(
-                    new RowDescriptor(descs, {tnode.hash_join_node.voutput_tuple_id}, {false}));
+            _output_row_desc = std::make_unique<RowDescriptor>(
+                    descs, std::vector<TTupleId> {tnode.hash_join_node.voutput_tuple_id},
+                    std::vector<bool> {false});
         }
     } else if (tnode.__isset.nested_loop_join_node) {
-        _intermediate_row_desc.reset(new RowDescriptor(
+        _intermediate_row_desc = std::make_unique<RowDescriptor>(
                 descs, tnode.nested_loop_join_node.vintermediate_tuple_id_list,
-                std::vector<bool>(tnode.nested_loop_join_node.vintermediate_tuple_id_list.size())));
+                std::vector<bool>(tnode.nested_loop_join_node.vintermediate_tuple_id_list.size()));
         if (!Base::_output_row_descriptor) {
-            _output_row_desc.reset(new RowDescriptor(
-                    descs, {tnode.nested_loop_join_node.voutput_tuple_id}, {false}));
+            _output_row_desc = std::make_unique<RowDescriptor>(
+                    descs, std::vector<TTupleId> {tnode.nested_loop_join_node.voutput_tuple_id},
+                    std::vector<bool> {false});
         }
     } else {
         // Iff BE has been upgraded and FE has not yet, we should keep origin logics for CROSS JOIN.
diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp
index 3186d4914bcf5d2..b911f28370f925f 100644
--- a/be/src/service/internal_service.cpp
+++ b/be/src/service/internal_service.cpp
@@ -616,7 +616,7 @@ void PInternalService::cancel_plan_fragment(google::protobuf::RpcController* /*c
         // Convert PPlanFragmentCancelReason to Status
         if (has_cancel_status) {
             // If fe set cancel status, then it is new FE now, should use cancel status.
-            actual_cancel_status = Status::create(request->cancel_status());
+            actual_cancel_status = Status::create<false>(request->cancel_status());
         } else if (has_cancel_reason) {
             // If fe not set cancel status, but set cancel reason, should convert cancel reason
             // to cancel status here.
diff --git a/be/src/vec/core/block.cpp b/be/src/vec/core/block.cpp
index 951c2661faf1727..0a1e658de753daa 100644
--- a/be/src/vec/core/block.cpp
+++ b/be/src/vec/core/block.cpp
@@ -802,26 +802,25 @@ void Block::update_hash(SipHash& hash) const {
 void Block::filter_block_internal(Block* block, const std::vector<uint32_t>& columns_to_filter,
                                   const IColumn::Filter& filter) {
     size_t count = filter.size() - simd::count_zero_num((int8_t*)filter.data(), filter.size());
-    if (count == 0) {
-        for (const auto& col : columns_to_filter) {
-            std::move(*block->get_by_position(col).column).assume_mutable()->clear();
+    for (const auto& col : columns_to_filter) {
+        auto& column = block->get_by_position(col).column;
+        if (column->size() == count) {
+            continue;
         }
-    } else {
-        for (const auto& col : columns_to_filter) {
-            auto& column = block->get_by_position(col).column;
-            if (column->size() != count) {
-                if (column->is_exclusive()) {
-                    const auto result_size = column->assume_mutable()->filter(filter);
-                    if (result_size != count) [[unlikely]] {
-                        throw Exception(ErrorCode::INTERNAL_ERROR,
-                                        "result_size not equal with filter_size, result_size={}, "
-                                        "filter_size={}",
-                                        result_size, count);
-                    }
-                } else {
-                    column = column->filter(filter, count);
-                }
+        if (count == 0) {
+            block->get_by_position(col).column->assume_mutable()->clear();
+            continue;
+        }
+        if (column->is_exclusive()) {
+            const auto result_size = column->assume_mutable()->filter(filter);
+            if (result_size != count) [[unlikely]] {
+                throw Exception(ErrorCode::INTERNAL_ERROR,
+                                "result_size not equal with filter_size, result_size={}, "
+                                "filter_size={}",
+                                result_size, count);
             }
+        } else {
+            column = column->filter(filter, count);
         }
     }
 }
diff --git a/be/src/vec/exprs/vexpr_context.cpp b/be/src/vec/exprs/vexpr_context.cpp
index b01ce2fade36602..3d909a5e56a2f9d 100644
--- a/be/src/vec/exprs/vexpr_context.cpp
+++ b/be/src/vec/exprs/vexpr_context.cpp
@@ -276,11 +276,12 @@ Status VExprContext::execute_conjuncts(const VExprContextSPtrs& conjuncts, Block
     if (rows == 0) {
         return Status::OK();
     }
+    if (null_map.size() != rows) {
+        return Status::InternalError("null_map.size() != rows, null_map.size()={}, rows={}",
+                                     null_map.size(), rows);
+    }
 
-    null_map.resize(rows);
     auto* final_null_map = null_map.get_data().data();
-    memset(final_null_map, 0, rows);
-    filter.resize_fill(rows, 1);
     auto* final_filter_ptr = filter.data();
 
     for (const auto& conjunct : conjuncts) {
@@ -326,8 +327,7 @@ Status VExprContext::execute_conjuncts_and_filter_block(const VExprContextSPtrs&
             execute_conjuncts(ctxs, nullptr, false, block, &result_filter, &can_filter_all));
     if (can_filter_all) {
         for (auto& col : columns_to_filter) {
-            // NOLINTNEXTLINE(performance-move-const-arg)
-            std::move(*block->get_by_position(col).column).assume_mutable()->clear();
+            block->get_by_position(col).column->assume_mutable()->clear();
         }
     } else {
         try {

From 83226838bb1a860f4dd3be26747e5d46a8662a38 Mon Sep 17 00:00:00 2001
From: Xin Liao <liaoxin@selectdb.com>
Date: Wed, 8 Jan 2025 11:54:02 +0800
Subject: [PATCH 041/140] [opt](load) Increase the default value of
 webserver_num_workers (#46304)

1. In high-concurrency stream load scenarios, there aren't enough
webserver threads available.
2. During stream load, after data reception and while waiting for
execution, the synchronous wait occupies HTTP threads.

To address the issue of insufficient HTTP threads during
high-concurrency stream loads causing slow data reception, we increased
the number of threads.
---
 be/src/common/config.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 82bfa49e3054488..29158c56bb50701 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -529,7 +529,7 @@ DEFINE_String(ssl_private_key_path, "");
 // Whether to check authorization
 DEFINE_Bool(enable_all_http_auth, "false");
 // Number of webserver workers
-DEFINE_Int32(webserver_num_workers, "48");
+DEFINE_Int32(webserver_num_workers, "128");
 
 DEFINE_Bool(enable_single_replica_load, "true");
 // Number of download workers for single replica load

From 2bbc9ac268014912719545f6371de800eee78685 Mon Sep 17 00:00:00 2001
From: shuke <shuke@selectdb.com>
Date: Wed, 8 Jan 2025 11:54:13 +0800
Subject: [PATCH 042/140] [regression-test](fix) test_audit_log_behavior.groovy
 case bug (#46588)

---
 regression-test/suites/audit/test_audit_log_behavior.groovy | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/regression-test/suites/audit/test_audit_log_behavior.groovy b/regression-test/suites/audit/test_audit_log_behavior.groovy
index ec01870750afa5e..6b6e7cf58718e5c 100644
--- a/regression-test/suites/audit/test_audit_log_behavior.groovy
+++ b/regression-test/suites/audit/test_audit_log_behavior.groovy
@@ -93,14 +93,15 @@ suite("test_audit_log_behavior") {
         for (int i = 0; i < cnt; i++) {
             def tuple2 = sqls.get(i)
             def retry = 180
-            def res = sql "select stmt from __internal_schema.audit_log where stmt like 'insert%3F6B9A_${i}%' order by time asc limit 1"
+            def query = "select stmt from __internal_schema.audit_log where stmt like 'insert%3F6B9A_${i}%' order by time asc limit 1"
+            def res = sql "${query}"
             while (res.isEmpty()) {
                 if (retry-- < 0) {
                     logger.warn("It has retried a few but still failed, you need to check it")
                     return
                 }
                 sleep(1000)
-                res = sql "select stmt from __internal_schema.audit_log where stmt like '%3F6B9A_${i}%' order by time asc limit 1"
+                res = sql "${query}"
             }
             assertEquals(res[0][0].toString(), tuple2[1].toString())
         }

From d541496cff053671eb94065e6c40cf6d11e7e8b2 Mon Sep 17 00:00:00 2001
From: zy-kkk <zhongyk10@gmail.com>
Date: Wed, 8 Jan 2025 12:00:21 +0800
Subject: [PATCH 043/140] [fix](test) Optimize the health check after oceanbase
 docker starts (#46434)

Change the check statement to the user and table used in the test
---
 docker/thirdparties/docker-compose/oceanbase/oceanbase.yaml.tpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/thirdparties/docker-compose/oceanbase/oceanbase.yaml.tpl b/docker/thirdparties/docker-compose/oceanbase/oceanbase.yaml.tpl
index 6dc507bcec6c8cf..e4f00a6a115e626 100644
--- a/docker/thirdparties/docker-compose/oceanbase/oceanbase.yaml.tpl
+++ b/docker/thirdparties/docker-compose/oceanbase/oceanbase.yaml.tpl
@@ -28,7 +28,7 @@ services:
     ports:
       - ${DOCKER_OCEANBASE_EXTERNAL_PORT}:2881
     healthcheck:
-      test: ["CMD-SHELL", "obclient -h127.1 -uroot@sys -P2881  -e 'SELECT 1'"]
+      test: ["CMD-SHELL", "obclient -h127.1 -P2881 -uroot@test -e 'SELECT * FROM doris_test.all_types limit 1'"]
       interval: 5s
       timeout: 60s
       retries: 120

From 8661311ab37f71b2b4cf87026be70ba027ecc740 Mon Sep 17 00:00:00 2001
From: zy-kkk <zhongyk10@gmail.com>
Date: Wed, 8 Jan 2025 12:08:02 +0800
Subject: [PATCH 044/140] [opt](jni) Modify Exception Cause Formatting in
 JniUtil for Better Readability (#46437)

This PR modifies the formatting of exception causes in the JniUtil
class. Previously, the causes were separated by a newline (\n). Now,
they are separated by | for better readability and consistent log
formatting.
---
 .../main/java/org/apache/doris/common/jni/utils/JniUtil.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JniUtil.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JniUtil.java
index 1a91df84be6904d..86d659a09f8329c 100644
--- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JniUtil.java
+++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JniUtil.java
@@ -72,7 +72,7 @@ public static String throwableToString(Throwable t) {
         // Follow the chain of exception causes and print them as well.
         Throwable cause = t;
         while ((cause = cause.getCause()) != null) {
-            output.write(String.format("\nCAUSED BY: %s: %s",
+            output.write(String.format(" | CAUSED BY: %s: %s",
                     cause.getClass().getSimpleName(), cause.getMessage()));
         }
         return output.toString();

From 9c7c0041836368fb7a0e55b5344c13fa11e3e708 Mon Sep 17 00:00:00 2001
From: Lei Zhang <zhanglei@selectdb.com>
Date: Wed, 8 Jan 2025 14:56:28 +0800
Subject: [PATCH 045/140] =?UTF-8?q?[fix](fe)=20Ignore=20`removeDroppedFron?=
 =?UTF-8?q?tends`=20when=20fe=20using=20metadata=20fail=E2=80=A6=20(#46446?=
 =?UTF-8?q?)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…ure recovery

* When use `metadata failure recovery` for changing fe ip, met exception
as follow:

```
ERROR (stateListener|29) [Env.transferToMaster():1729] failed to transfer to master. progress: replay journal
com.sleepycat.je.rep.UnknownMasterException: (JE 18.3.12) Could not determine master from helpers at:[/172.59.0.1:12010]
        at com.sleepycat.je.rep.elections.Learner.findMaster(Learner.java:443) ~[je-18.3.14-doris-SNAPSHOT.jar:18.3.14-doris-SNAPSHOT]
        at com.sleepycat.je.rep.util.ReplicationGroupAdmin.getMasterSocket(ReplicationGroupAdmin.java:186) ~[je-18.3.14-doris-SNAPSHOT.jar:18.3.14-doris-SNAPSHOT]
        at com.sleepycat.je.rep.util.ReplicationGroupAdmin.doMessageExchange(ReplicationGroupAdmin.java:607) ~[je-18.3.14-doris-SNAPSHOT.jar:18.3.14-doris-SNAPSHOT]
        at com.sleepycat.je.rep.util.ReplicationGroupAdmin.getGroup(ReplicationGroupAdmin.java:406) ~[je-18.3.14-doris-SNAPSHOT.jar:18.3.14-doris-SNAPSHOT]
        at org.apache.doris.ha.BDBHA.removeDroppedMember(BDBHA.java:256) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.catalog.Env.removeDroppedFrontends(Env.java:3184) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.catalog.Env.transferToMaster(Env.java:1597) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.catalog.Env.access$1200(Env.java:351) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.catalog.Env$5.runOneCycle(Env.java:2885) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.common.util.Daemon.run(Daemon.java:119) ~[doris-fe.jar:1.2-SNAPSHOT]
```
---
 fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 505970e104c26f6..7e0617a84a0a541 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -3179,6 +3179,13 @@ public void dropFrontendFromBDBJE(FrontendNodeType role, String host, int port)
     }
 
     private void removeDroppedFrontends(ConcurrentLinkedQueue<String> removedFrontends) {
+        if (!Strings.isNullOrEmpty(System.getProperty(FeConstants.METADATA_FAILURE_RECOVERY_KEY))) {
+            // metadata recovery mode
+            LOG.info("Metadata failure recovery({}), ignore removing dropped frontends",
+                    System.getProperty(FeConstants.METADATA_FAILURE_RECOVERY_KEY));
+            return;
+        }
+
         if (haProtocol != null && haProtocol instanceof BDBHA) {
             BDBHA bdbha = (BDBHA) haProtocol;
             bdbha.removeDroppedMember(removedFrontends);

From 96e4e9e5640e2f3397de34b143a4990ddb136bb1 Mon Sep 17 00:00:00 2001
From: Xin Liao <liaoxin@selectdb.com>
Date: Wed, 8 Jan 2025 15:04:39 +0800
Subject: [PATCH 046/140] [Fix](mow) Multiple table locks should be acquired in
 order of table ids (#46305)

Lock acquisition for multiple tables needs to be sorted by table ids, as
unordered locking can result in deadlocks. Even though the table list is
already sorted when received, we sort it again here as a defensive
programming measure to prevent issues from future code modifications.
---
 .../doris/cloud/transaction/CloudGlobalTransactionMgr.java   | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
index ffe32348bd09159..2ff882bcbfc9d7e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
@@ -137,6 +137,7 @@
 import java.security.SecureRandom;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -1167,8 +1168,12 @@ private void beforeCommitTransaction(List<Table> tableList, long transactionId,
             }
         }
 
+        // Get tables that require commit lock - only MOW tables need this:
+        // 1. Filter to keep only OlapTables with MOW enabled
+        // 2. Sort by table ID to maintain consistent locking order and prevent deadlocks
         List<Table> mowTableList = tableList.stream()
                 .filter(table -> table instanceof OlapTable && ((OlapTable) table).getEnableUniqueKeyMergeOnWrite())
+                .sorted(Comparator.comparingLong(Table::getId))
                 .collect(Collectors.toList());
         increaseWaitingLockCount(mowTableList);
         if (!MetaLockUtils.tryCommitLockTables(mowTableList, timeoutMillis, TimeUnit.MILLISECONDS)) {

From 744691a0f3e8fa76c5d0d0d2a8188b259830ce10 Mon Sep 17 00:00:00 2001
From: Gabriel <liwenqiang@selectdb.com>
Date: Wed, 8 Jan 2025 15:07:15 +0800
Subject: [PATCH 047/140] [fix](set operator) Fix invalid access in set
 operator (#46528)

Introduced by #45207.
For a query plan,
```
set sink operator -- -|
                                    |-> set source operator
set probe operator --|
```
If `set source operators` are finished (due to limit reached), all
upstream operators could be finished and waken up at the same time.
However, some states are initialized in `set sink operator`. So if `set
probe operator` executes before `set sink operator` initialization, it
will incur an invalid address access.

*** Query id: cebb723bbda64249-9ab8c9e7aa72c540 ***
*** is nereids: 1 ***
*** tablet id: 0 ***
*** Aborted at 1736092087 (unix time) try "date -d @1736092087" if you
are using GNU date ***
*** Current BE git commitID: 26d68d778a ***
*** SIGSEGV address not mapped to object (@0xf8) received by PID 23579
(TID 26524 OR 0x7f84d4240640) from PID 248; stack trace: ***
0# doris::signal::(anonymous namespace)::FailureSignalHandler(int,
siginfo_t*, void*) at
/home/zcp/repo_center/doris_master/doris/be/src/common/signal_handler.h:421
1# PosixSignals::chained_handler(int, siginfo*, void*) [clone .part.0]
in /usr/lib/jvm/java-17-openjdk-amd64/lib/server/libjvm.so
2# JVM_handle_linux_signal in
/usr/lib/jvm/java-17-openjdk-amd64/lib/server/libjvm.so
 3# 0x00007F88F4063520 in /lib/x86_64-linux-gnu/libc.so.6
4#
doris::pipeline::SetProbeSinkOperatorX::_finalize_probe(doris::pipeline::SetProbeSinkLocalState&)
at
/home/zcp/repo_center/doris_master/doris/be/src/pipeline/exec/set_probe_sink_operator.cpp:183
5# doris::pipeline::SetProbeSinkOperatorX::sink(doris::RuntimeState*,
doris::vectorized::Block*, bool) at
/home/zcp/repo_center/doris_master/doris/be/src/pipeline/exec/set_probe_sink_operator.cpp:99
6# doris::pipeline::PipelineTask::execute(bool*) at
/home/zcp/repo_center/doris_master/doris/be/src/pipeline/pipeline_task.cpp:383
7# doris::pipeline::TaskScheduler::_do_work(int) at
/home/zcp/repo_center/doris_master/doris/be/src/pipeline/task_scheduler.cpp:138
8# doris::ThreadPool::dispatch_thread() in
/mnt/hdd01/ci/master-deploy/be/lib/doris_be
9# doris::Thread::supervise_thread(void*) at
/home/zcp/repo_center/doris_master/doris/be/src/util/thread.cpp:499
10# start_thread at ./nptl/pthread_create.c:442
11# 0x00007F88F4147850 at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:83
---
 be/src/pipeline/dependency.h                     | 3 ++-
 be/src/pipeline/exec/set_probe_sink_operator.cpp | 7 +++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/be/src/pipeline/dependency.h b/be/src/pipeline/dependency.h
index fd179cdfd0a7977..1b811219c322e9d 100644
--- a/be/src/pipeline/dependency.h
+++ b/be/src/pipeline/dependency.h
@@ -660,7 +660,8 @@ struct SetSharedState : public BasicSharedState {
     //// shared static states (shared, decided in prepare/open...)
 
     /// init in setup_local_state
-    std::unique_ptr<SetDataVariants> hash_table_variants = nullptr; // the real data HERE.
+    std::unique_ptr<SetDataVariants> hash_table_variants =
+            std::make_unique<SetDataVariants>(); // the real data HERE.
     std::vector<bool> build_not_ignore_null;
 
     // The SET operator's child might have different nullable attributes.
diff --git a/be/src/pipeline/exec/set_probe_sink_operator.cpp b/be/src/pipeline/exec/set_probe_sink_operator.cpp
index db487b0f9e72521..929062ec80b6c5e 100644
--- a/be/src/pipeline/exec/set_probe_sink_operator.cpp
+++ b/be/src/pipeline/exec/set_probe_sink_operator.cpp
@@ -86,8 +86,8 @@ Status SetProbeSinkOperatorX<is_intersect>::sink(RuntimeState* state, vectorized
                                 process_hashtable_ctx(&local_state, probe_rows);
                         return process_hashtable_ctx.mark_data_in_hashtable(arg);
                     } else {
-                        LOG(FATAL) << "FATAL: uninited hash table";
-                        __builtin_unreachable();
+                        LOG(WARNING) << "Uninited hash table in Set Probe Sink Operator";
+                        return Status::OK();
                     }
                 },
                 local_state._shared_state->hash_table_variants->method_variant));
@@ -258,8 +258,7 @@ void SetProbeSinkOperatorX<is_intersect>::_refresh_hash_table(
 
                     arg.reset();
                 } else {
-                    LOG(FATAL) << "FATAL: uninited hash table";
-                    __builtin_unreachable();
+                    LOG(WARNING) << "Uninited hash table in Set Probe Sink Operator";
                 }
             },
             hash_table_variants->method_variant);

From 5a28a4b719dfdbc4ea7b89aad80a207e7f6586d7 Mon Sep 17 00:00:00 2001
From: zgxme <zhenggaoxiong@selectdb.com>
Date: Wed, 8 Jan 2025 17:47:51 +0800
Subject: [PATCH 048/140] [chore](regression) modify cos bucket of paimon case
 (#46592)

### What problem does this PR solve?
That previous bucket is no longer used and configured for life cycle.
Replaced with a bucket for external testing.
---
 .../external_table_p0/paimon/paimon_base_filesystem.groovy    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy b/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy
index 79df788f28dbdb1..af5134a3b4615f6 100644
--- a/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy
+++ b/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy
@@ -48,7 +48,7 @@ suite("paimon_base_filesystem", "p0,external,doris,external_docker,external_dock
             create catalog if not exists ${catalog_cos} properties (
                 "type" = "paimon",
                 "paimon.catalog.type"="filesystem",
-                "warehouse" = "s3://doris-build-1308700295/regression/paimon1",
+                "warehouse" = "s3://sdb-qa-datalake-test-1308700295/regression/paimon1",
                 "s3.access_key" = "${txYunAk}",
                 "s3.secret_key" = "${txYunSk}",
                 "s3.endpoint" = "cos.ap-beijing.myqcloud.com"
@@ -58,7 +58,7 @@ suite("paimon_base_filesystem", "p0,external,doris,external_docker,external_dock
             create catalog if not exists ${catalog_cosn} properties (
                 "type" = "paimon",
                 "paimon.catalog.type"="filesystem",
-                "warehouse" = "cosn://doris-build-1308700295/regression/paimon1",
+                "warehouse" = "cosn://sdb-qa-datalake-test-1308700295/regression/paimon1",
                 "cos.access_key" = "${txYunAk}",
                 "cos.secret_key" = "${txYunSk}",
                 "cos.endpoint" = "cos.ap-beijing.myqcloud.com"

From 57e0a1cf13ad93aaca560b3e91d8e1030ee4c630 Mon Sep 17 00:00:00 2001
From: minghong <zhouminghong@selectdb.com>
Date: Wed, 8 Jan 2025 18:31:25 +0800
Subject: [PATCH 049/140] [fix](nereids) fix bug in PhysicalTopN.equals()
 (#46547)

---
 .../trees/plans/physical/PhysicalTopN.java    |  4 +-
 .../plans/physical/PhysicalTopNTest.java      | 55 +++++++++++++++++++
 2 files changed, 58 insertions(+), 1 deletion(-)
 create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTopNTest.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTopN.java
index 5f59f8a7083e1f0..014daa7bbcf7780 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTopN.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTopN.java
@@ -93,7 +93,9 @@ public boolean equals(Object o) {
             return false;
         }
         PhysicalTopN<?> that = (PhysicalTopN<?>) o;
-        return limit == that.limit && offset == that.offset;
+        return limit == that.limit && offset == that.offset
+                && this.phase == that.phase
+                && Objects.equals(that.getOrderKeys(), getOrderKeys());
     }
 
     @Override
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTopNTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTopNTest.java
new file mode 100644
index 000000000000000..c411ad44619cc87
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTopNTest.java
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.physical;
+
+import org.apache.doris.nereids.properties.OrderKey;
+import org.apache.doris.nereids.trees.expressions.ExprId;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.SortPhase;
+import org.apache.doris.nereids.types.BigIntType;
+
+import com.google.common.collect.Lists;
+import mockit.Mocked;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+public class PhysicalTopNTest {
+    @Test
+    public void testEquals(@Mocked Plan child) {
+        SlotReference a = new SlotReference(new ExprId(0), "a",
+                BigIntType.INSTANCE, true, Lists.newArrayList());
+        List<OrderKey> orderKeysA = Lists.newArrayList();
+        orderKeysA.add(new OrderKey(a, true, true));
+        PhysicalTopN topn1 = new PhysicalTopN(orderKeysA, 1, 1, SortPhase.LOCAL_SORT,
+                null, child);
+        PhysicalTopN topn2 = new PhysicalTopN(orderKeysA, 1, 1, SortPhase.GATHER_SORT,
+                null, child);
+        Assertions.assertNotEquals(topn1, topn2);
+
+        SlotReference b = new SlotReference(new ExprId(0), "b",
+                BigIntType.INSTANCE, true, Lists.newArrayList());
+        List<OrderKey> orderKeysB = Lists.newArrayList();
+        orderKeysB.add(new OrderKey(b, true, true));
+        PhysicalTopN topn3 = new PhysicalTopN(orderKeysB, 1, 1, SortPhase.LOCAL_SORT,
+                null, child);
+        Assertions.assertNotEquals(topn2, topn3);
+    }
+}

From 840532152ec94f8f68a5e8db0e27f2474b28e8a4 Mon Sep 17 00:00:00 2001
From: Lijia Liu <liutang123@yeah.net>
Date: Wed, 8 Jan 2025 19:24:38 +0800
Subject: [PATCH 050/140] [fix](planner) fix core when select and filter by
 slot in old planner (#46541)

### What problem does this PR solve?

Problem Summary:
In 2.1.7, if a sql parse failed in nereids planner, it will fallback to old planner.
and old planner maybe create `SelectNode` with some conjuncts that are
not boolean type.

Reproduce:
Create table sql see #46498
query sql1:
select b.c_id   from
dbgr as b
left join
(select c.c_id from lo  where event_date between 20220500 and 20220600 limit 100 )c
on c.c_id LIMIT  200;

query sql2:
select b.c_id
from dbgr as b
left join
(select c.c_id from lo )c
on c.c_id
LIMIT 0, 200;

Because `select c.c_id`, these sqls will fallback to old planner.
Because `on c.c_id` is not boolean type, and be will core.
A part of query plan is as follows:

|   1:VOlapScanNode                                                           |
|      TABLE: test.lo(lo), PREAGGREGATION: ON                                 |
|      PREDICATES: (`c`.`c_id` AND (`test`.`lo`.`__DORIS_DELETE_SIGN__` = 0)) |
|      partitions=1/3 (p_202206)                                              |
|      tablets=2/2, tabletList=89678,89680                                    |
|      cardinality=46, avgRowSize=165.54349, numNodes=1                       |
|      pushAggOp=NONE                                                         |
+-----------------------------------------------------------------------------+

A fatal log is as follows:

F20241219 23:13:23.457937 33282 assert_cast.h:58] Bad cast from type:doris::vectorized::ColumnVector<int> to doris::vectorized::ColumnVector<unsigned
char>
*** Check failure stack trace: ***
    @     0x55bfa043b956  google::LogMessageFatal::~LogMessageFatal()
    @     0x55bf6f3bc070  assert_cast<>()
    @     0x55bf8978d767  doris::vectorized::VExprContext::execute_conjuncts()
    @     0x55bf8978c463  doris::vectorized::VExprContext::execute_conjuncts_and_filter_block()
    @     0x55bf8978bf72  doris::vectorized::VExprContext::filter_block()
    @     0x55bfa035b8e4  doris::pipeline::SelectOperatorX::pull()
    @     0x55bf9fee2b62  doris::pipeline::StreamingOperatorX<>::get_block()
    @     0x55bf9feab54b  doris::pipeline::OperatorXBase::get_block_after_projects()
    @     0x55bfa03dd07c  doris::pipeline::PipelineXTask::execute()
    @     0x55bfa0413e85  doris::pipeline::TaskScheduler::_do_work()
    @     0x55bfa0417dcb  doris::pipeline::TaskScheduler::start()::$_0::operator()()
    @     0x55bfa0417d55  std::__invoke_impl<>()
    @     0x55bfa0417d05  _ZSt10__invoke_rIvRZN5doris8pipeline13TaskScheduler5startEvE3$_0JEENSt9enable_ifIX16is_invocable_r_vIT_T0_DpT1_EES6_E4typeEO
S7_DpOS8_
    @     0x55bfa0417bcd  std::_Function_handler<>::_M_invoke()
    @     0x55bf6e6c9b63  std::function<>::operator()()
    @     0x55bf7289e209  doris::FunctionRunnable::run()
    @     0x55bf728899c0  doris::ThreadPool::dispatch_thread()
    @     0x55bf728b0c24  std::__invoke_impl<>()
    @     0x55bf728b0afd  std::__invoke<>()
    @     0x55bf728b0a85  _ZNSt5_BindIFMN5doris10ThreadPoolEFvvEPS1_EE6__callIvJEJLm0EEEET_OSt5tupleIJDpT0_EESt12_Index_tupleIJXspT1_EEE
    @     0x55bf728b092e  std::_Bind<>::operator()<>()
    @     0x55bf728b0845  std::__invoke_impl<>()
    @     0x55bf728b07e5  _ZSt10__invoke_rIvRSt5_BindIFMN5doris10ThreadPoolEFvvEPS2_EEJEENSt9enable_ifIX16is_invocable_r_vIT_T0_DpT1_EESA_E4typeEOSB_D
pOSC_
    @     0x55bf728b048d  std::_Function_handler<>::_M_invoke()
    @     0x55bf6e6c9b63  std::function<>::operator()()
    @     0x55bf728521fc  doris::Thread::supervise_thread()
    @     0x7f4260614ea5  start_thread
    @     0x7f42610439fd  __clone
    @              (nil)  (unknown)

And another:

F20250108 13:07:05.275424 184257 assert_cast.h:58] Bad cast from type:doris::vectorized::ColumnVector<int> to doris::vectorized::ColumnVector<unsigned char>
0# doris::signal::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*) at /root/tc/be/src/common/signal_handler.h:421
 1# 0x00007FB73FB31400 in /lib64/libc.so.6
 2# __GI_raise in /lib64/libc.so.6
 3# abort in /lib64/libc.so.6
 4# 0x000055CDAAF0090D in /usr/local/service/doris/lib/be/doris_be
 5# google::LogMessage::SendToLog() in /usr/local/service/doris/lib/be/doris_be
 6# google::LogMessage::Flush() in /usr/local/service/doris/lib/be/doris_be
 7# google::LogMessageFatal::~LogMessageFatal() in /usr/local/service/doris/lib/be/doris_be
 8# doris::vectorized::ColumnVector<unsigned char> const& assert_cast<doris::vectorized::ColumnVector<unsigned char> const&, doris::vec
torized::IColumn const&>(doris::vectorized::IColumn const&) in /usr/local/service/doris/lib/be/doris_be
 9# doris::vectorized::VExprContext::execute_conjuncts(std::vector<std::shared_ptr<doris::vectorized::VExprContext>, std::allocator<std
::shared_ptr<doris::vectorized::VExprContext> > > const&, std::vector<doris::vectorized::PODArray<unsigned char, 4096ul, Allocator<fals
e, false, false, DefaultMemoryAllocator>, 16ul, 16ul>*, std::allocator<doris::vectorized::PODArray<unsigned char, 4096ul, Allocator<fal
se, false, false, DefaultMemoryAllocator>, 16ul, 16ul>*> > const*, bool, doris::vectorized::Block*, doris::vectorized::PODArray<unsigne
d char, 4096ul, Allocator<false, false, false, DefaultMemoryAllocator>, 16ul, 16ul>*, bool*) at /root/tc/be/src/vec/exprs/vexpr_context
.cpp:181
10# doris::vectorized::VExprContext::execute_conjuncts_and_filter_block(std::vector<std::shared_ptr<doris::vectorized::VExprContext>, s
td::allocator<std::shared_ptr<doris::vectorized::VExprContext> > > const&, doris::vectorized::Block*, std::vector<unsigned int, std::al
locator<unsigned int> >&, int, doris::vectorized::PODArray<unsigned char, 4096ul, Allocator<false, false, false, DefaultMemoryAllocator
>, 16ul, 16ul>&) at /root/tc/be/src/vec/exprs/vexpr_context.cpp:324
11# doris::segment_v2::SegmentIterator::_execute_common_expr(unsigned short*, unsigned short&, doris::vectorized::Block*) at /root/tc/b
e/src/olap/rowset/segment_v2/segment_iterator.cpp:2274
12# doris::segment_v2::SegmentIterator::_next_batch_internal(doris::vectorized::Block*) at /root/tc/be/src/olap/rowset/segment_v2/segment_iterator.cpp:2178
13# doris::segment_v2::SegmentIterator::next_batch(doris::vectorized::Block*)::$_0::operator()() const at /root/tc/be/src/olap/rowset/segment_v2/segment_iterator.cpp:1914
14# doris::segment_v2::SegmentIterator::next_batch(doris::vectorized::Block*) at /root/tc/be/src/olap/rowset/segment_v2/segment_iterator.cpp:1913
15# doris::segment_v2::LazyInitSegmentIterator::next_batch(doris::vectorized::Block*) in /usr/local/service/doris/lib/be/doris_be
16# doris::BetaRowsetReader::next_block(doris::vectorized::Block*) at /root/tc/be/src/olap/rowset/beta_rowset_reader.cpp:348
17# doris::vectorized::VCollectIterator::Level0Iterator::_refresh() in /usr/local/service/doris/lib/be/doris_be
18# doris::vectorized::VCollectIterator::Level0Iterator::refresh_current_row() at /root/tc/be/src/vec/olap/vcollect_iterator.cpp:511
19# doris::vectorized::VCollectIterator::Level0Iterator::ensure_first_row_ref() at /root/tc/be/src/vec/olap/vcollect_iterator.cpp:482
20# doris::vectorized::VCollectIterator::Level1Iterator::ensure_first_row_ref() at /root/tc/be/src/vec/olap/vcollect_iterator.cpp:697
21# doris::vectorized::VCollectIterator::build_heap(std::vector<std::shared_ptr<doris::RowsetReader>, std::allocator<std::shared_ptr<doris::RowsetReader> > >&) at /root/tc/be/src/vec/olap/vcollect_iterator.cpp:186
22# doris::vectorized::BlockReader::_init_collect_iter(doris::TabletReader::ReaderParams const&) at /root/tc/be/src/vec/olap/block_reader.cpp:139
23# doris::vectorized::BlockReader::init(doris::TabletReader::ReaderParams const&) at /root/tc/be/src/vec/olap/block_reader.cpp:211
24# doris::vectorized::NewOlapScanner::open(doris::RuntimeState*) at /root/tc/be/src/vec/exec/scan/new_olap_scanner.cpp:227
25# doris::vectorized::ScannerScheduler::_scanner_scan(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>) at /root/tc/be/src/vec/exec/scan/scanner_scheduler.cpp:259
26# doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::{lambda()#1}::operator()() const::{lambda()#2}::operator()() const at /root/tc/be/src/vec/exec/scan/scanner_scheduler.cpp:180
...

---------

Co-authored-by: liutang123 <liulijia@gmail.com>
---
 .../org/apache/doris/planner/PlanNode.java    | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
index 73768435154b761..c83cd9cde99b999 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
@@ -34,6 +34,7 @@
 import org.apache.doris.analysis.TupleId;
 import org.apache.doris.catalog.Column;
 import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.Type;
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.Id;
 import org.apache.doris.common.NotImplementedException;
@@ -59,6 +60,8 @@
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -89,6 +92,7 @@
  * its children (= are bound by tupleIds).
  */
 public abstract class PlanNode extends TreeNode<PlanNode> implements PlanStats {
+    private static final Logger LOG = LogManager.getLogger(PlanNode.class);
 
     protected String planNodeName;
 
@@ -207,6 +211,7 @@ protected PlanNode(PlanNodeId id, PlanNode node, String planNodeName, Statistica
         this.tblRefIds = Lists.newArrayList(node.tblRefIds);
         this.nullableTupleIds = Sets.newHashSet(node.nullableTupleIds);
         this.conjuncts = Expr.cloneList(node.conjuncts, null);
+
         this.cardinality = -1;
         this.compactData = node.compactData;
         this.planNodeName = "V" + planNodeName;
@@ -806,6 +811,21 @@ public void init() throws UserException {}
     public void init(Analyzer analyzer) throws UserException {
         assignConjuncts(analyzer);
         createDefaultSmap(analyzer);
+        castConjuncts();
+    }
+
+    private void castConjuncts() throws AnalysisException {
+        for (int i = 0; i < conjuncts.size(); ++i) {
+            Expr expr = conjuncts.get(i);
+            if (!expr.getType().isBoolean()) {
+                try {
+                    conjuncts.set(i, expr.castTo(Type.BOOLEAN));
+                } catch (AnalysisException e) {
+                    LOG.warn("{} is not boolean and can not be cast to boolean", expr.toSql(), e);
+                    throw new AnalysisException("conjuncts " + expr.toSql() + " is not boolean");
+                }
+            }
+        }
     }
 
     /**

From 029686f43325fb3f57e56825884fe5de07a739a2 Mon Sep 17 00:00:00 2001
From: minghong <zhouminghong@selectdb.com>
Date: Wed, 8 Jan 2025 22:10:41 +0800
Subject: [PATCH 051/140] [opt](nereids) let Physical plan compute
 logicalProperties.datatrait (#46401)

### What problem does this PR solve?
logicalProperties.datatrait is not only used by LogicalPlan, it is also
used in postprocessor, especially used in RuntimeFilterGeneration.
T1 join T2 on T1.a=T2.a where T1.a=1 and T2.a=1
Because both T1.a and T2.a are unique value "1", the RF: T1.a->T2.a is
useless
---
 .../post/RuntimeFilterGenerator.java          |  31 ++++
 .../nereids/trees/plans/AbstractPlan.java     |   5 +-
 .../trees/plans/BlockFuncDepsPropagation.java |   3 +-
 .../doris/nereids/trees/plans/FakePlan.java   |   2 +-
 .../doris/nereids/trees/plans/Plan.java       |  49 +++++++
 .../trees/plans/PropagateFuncDeps.java        |   3 +-
 .../logical/LogicalDeferMaterializeTopN.java  |  21 +--
 .../trees/plans/logical/LogicalExcept.java    |   7 +-
 .../trees/plans/logical/LogicalPlan.java      |  53 -------
 .../trees/plans/logical/LogicalSink.java      |  21 +++
 .../plans/physical/PhysicalAssertNumRows.java |  31 ++++
 .../plans/physical/PhysicalCTEAnchor.java     |   4 +-
 .../plans/physical/PhysicalCTEConsumer.java   |   3 +-
 .../plans/physical/PhysicalCTEProducer.java   |   4 +-
 .../physical/PhysicalCatalogRelation.java     |  49 +++++++
 .../PhysicalDeferMaterializeTopN.java         |  29 ++++
 .../plans/physical/PhysicalDistribute.java    |   4 +-
 .../plans/physical/PhysicalEmptyRelation.java |   4 +-
 .../trees/plans/physical/PhysicalExcept.java  |  43 ++++++
 .../trees/plans/physical/PhysicalFilter.java  |  34 +++++
 .../plans/physical/PhysicalGenerate.java      |  21 +++
 .../plans/physical/PhysicalHashAggregate.java |  92 ++++++++++++
 .../plans/physical/PhysicalHashJoin.java      | 137 ++++++++++++++++++
 .../plans/physical/PhysicalIntersect.java     |  58 ++++++++
 .../trees/plans/physical/PhysicalLimit.java   |  29 ++++
 .../physical/PhysicalNestedLoopJoin.java      |   4 +-
 .../physical/PhysicalOneRowRelation.java      |  32 ++++
 .../plans/physical/PhysicalPartitionTopN.java |   4 +-
 .../trees/plans/physical/PhysicalProject.java |  72 +++++++++
 .../plans/physical/PhysicalQuickSort.java     |   4 +-
 .../trees/plans/physical/PhysicalRepeat.java  |  21 +++
 .../trees/plans/physical/PhysicalSink.java    |  21 +++
 .../plans/physical/PhysicalSqlCache.java      |   4 +-
 .../plans/physical/PhysicalTVFRelation.java   |   3 +-
 .../trees/plans/physical/PhysicalTopN.java    |  29 ++++
 .../trees/plans/physical/PhysicalUnion.java   |  96 ++++++++++++
 .../trees/plans/physical/PhysicalWindow.java  |  83 +++++++++++
 .../runtime_filter/runtime_filter.out         |  19 +++
 .../cte/test_cte_filter_pushdown.out          |   8 +-
 .../nereids_tpch_p0/tpch/runtime_filter.out   |  29 ++++
 .../runtime_filter/runtime_filter.groovy      |  38 +++++
 41 files changed, 1104 insertions(+), 100 deletions(-)
 create mode 100644 regression-test/data/nereids_p0/runtime_filter/runtime_filter.out
 create mode 100644 regression-test/data/nereids_tpch_p0/tpch/runtime_filter.out

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java
index c7e8ae01fcd75fb..f1ed71373eebfa7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java
@@ -73,6 +73,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -269,6 +270,9 @@ public PhysicalPlan visitPhysicalHashJoin(PhysicalHashJoin<? extends Plan, ? ext
         for (int i = 0; i < hashJoinConjuncts.size(); i++) {
             EqualPredicate equalTo = JoinUtils.swapEqualToForChildrenOrder(
                     (EqualPredicate) hashJoinConjuncts.get(i), join.left().getOutputSet());
+            if (isUniqueValueEqualTo(join, equalTo)) {
+                continue;
+            }
             for (TRuntimeFilterType type : legalTypes) {
                 //bitmap rf is generated by nested loop join.
                 if (type == TRuntimeFilterType.BITMAP) {
@@ -292,6 +296,33 @@ public PhysicalPlan visitPhysicalHashJoin(PhysicalHashJoin<? extends Plan, ? ext
         return join;
     }
 
+    /**
+     *
+     * T1 join T1 on T1.a=T2.a where T1.a=1 and T2.a=1
+     * if T1.a = T2.a, no need to generate RF by "T1.a=T2.a"
+     *
+     * T1 join T1 on T1.a=T2.a where T1.a in (1, 2) and T2.a in (1, 2)
+     * in above case RF: T2.a->T1.a is generated, because the limitation of
+     * const propagation
+     */
+    private boolean isUniqueValueEqualTo(PhysicalHashJoin<? extends Plan, ? extends Plan> join,
+                                         EqualPredicate equalTo) {
+        if (equalTo.left() instanceof Slot && equalTo.right() instanceof Slot) {
+            Optional<Expression> leftValue = join.left().getLogicalProperties()
+                    .getTrait().getUniformValue((Slot) equalTo.left());
+            Optional<Expression> rightValue = join.right().getLogicalProperties()
+                    .getTrait().getUniformValue((Slot) equalTo.right());
+            if (leftValue != null && rightValue != null) {
+                if (leftValue.isPresent() && rightValue.isPresent()) {
+                    if (leftValue.get().equals(rightValue.get())) {
+                        return true;
+                    }
+                }
+            }
+        }
+        return false;
+    }
+
     @Override
     public PhysicalCTEConsumer visitPhysicalCTEConsumer(PhysicalCTEConsumer scan, CascadesContext context) {
         RuntimeFilterContext ctx = context.getRuntimeFilterContext();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java
index 958b4fe9c424b4a..5b08bdd2e5b034e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java
@@ -27,7 +27,6 @@
 import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
 import org.apache.doris.nereids.trees.plans.TreeStringPlan.TreeStringNode;
-import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
 import org.apache.doris.nereids.util.MutableState;
 import org.apache.doris.nereids.util.TreeStringUtils;
 import org.apache.doris.statistics.Statistics;
@@ -202,9 +201,7 @@ public LogicalProperties computeLogicalProperties() {
             return UnboundLogicalProperties.INSTANCE;
         } else {
             Supplier<List<Slot>> outputSupplier = Suppliers.memoize(this::computeOutput);
-            Supplier<DataTrait> fdSupplier = () -> this instanceof LogicalPlan
-                    ? ((LogicalPlan) this).computeDataTrait()
-                    : DataTrait.EMPTY_TRAIT;
+            Supplier<DataTrait> fdSupplier = () -> computeDataTrait();
             return new LogicalProperties(outputSupplier, fdSupplier);
         }
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/BlockFuncDepsPropagation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/BlockFuncDepsPropagation.java
index f692b5b917f31b6..4af1132ae39f3a5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/BlockFuncDepsPropagation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/BlockFuncDepsPropagation.java
@@ -18,12 +18,11 @@
 package org.apache.doris.nereids.trees.plans;
 
 import org.apache.doris.nereids.properties.DataTrait;
-import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
 
 /**
  * Block fd propagation, it always returns an empty fd
  */
-public interface BlockFuncDepsPropagation extends LogicalPlan {
+public interface BlockFuncDepsPropagation extends Plan {
     @Override
     default DataTrait computeDataTrait() {
         return DataTrait.EMPTY_TRAIT;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java
index 9fcf1872116e04a..39c546cdb197cc1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java
@@ -37,7 +37,7 @@
 /**
  * Used for unit test only.
  */
-public class FakePlan implements Plan {
+public class FakePlan implements Plan, BlockFuncDepsPropagation {
     private MutableState mutableState = new MultiMutableState();
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java
index 60af0d18666f2bc..402ad046d910ce3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java
@@ -18,6 +18,7 @@
 package org.apache.doris.nereids.trees.plans;
 
 import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DataTrait;
 import org.apache.doris.nereids.properties.LogicalProperties;
 import org.apache.doris.nereids.properties.UnboundLogicalProperties;
 import org.apache.doris.nereids.trees.TreeNode;
@@ -232,4 +233,52 @@ default String getGroupIdAsString() {
     default String getGroupIdWithPrefix() {
         return "@" + getGroupIdAsString();
     }
+
+    /**
+     * Compute DataTrait for different plan
+     * Note: Unless you really know what you're doing, please use the following interface.
+     *   - BlockFDPropagation: clean the fd
+     *   - PropagateFD: propagate the fd
+     */
+    default DataTrait computeDataTrait() {
+        DataTrait.Builder fdBuilder = new DataTrait.Builder();
+        computeUniform(fdBuilder);
+        computeUnique(fdBuilder);
+        computeEqualSet(fdBuilder);
+        computeFd(fdBuilder);
+
+        for (Slot slot : getOutput()) {
+            Set<Slot> o = ImmutableSet.of(slot);
+            // all slots dependent unique slot
+            for (Set<Slot> uniqueSlot : fdBuilder.getAllUniqueAndNotNull()) {
+                fdBuilder.addDeps(uniqueSlot, o);
+            }
+            // uniform slot dependents all slots
+            for (Set<Slot> uniformSlot : fdBuilder.getAllUniformAndNotNull()) {
+                fdBuilder.addDeps(o, uniformSlot);
+            }
+        }
+        for (Set<Slot> equalSet : fdBuilder.calEqualSetList()) {
+            Set<Slot> validEqualSet = Sets.intersection(getOutputSet(), equalSet);
+            fdBuilder.addDepsByEqualSet(validEqualSet);
+            fdBuilder.addUniformByEqualSet(validEqualSet);
+            fdBuilder.addUniqueByEqualSet(validEqualSet);
+        }
+        Set<Slot> output = this.getOutputSet();
+        for (Plan child : children()) {
+            if (!output.containsAll(child.getOutputSet())) {
+                fdBuilder.pruneSlots(output);
+                break;
+            }
+        }
+        return fdBuilder.build();
+    }
+
+    void computeUnique(DataTrait.Builder builder);
+
+    void computeUniform(DataTrait.Builder builder);
+
+    void computeEqualSet(DataTrait.Builder builder);
+
+    void computeFd(DataTrait.Builder builder);
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PropagateFuncDeps.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PropagateFuncDeps.java
index c316ea2462e61af..92a240b50ad4609 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PropagateFuncDeps.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PropagateFuncDeps.java
@@ -18,12 +18,11 @@
 package org.apache.doris.nereids.trees.plans;
 
 import org.apache.doris.nereids.properties.DataTrait;
-import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
 
 /**
  * Propagate fd, keep children's fd
  */
-public interface PropagateFuncDeps extends LogicalPlan {
+public interface PropagateFuncDeps extends Plan {
     @Override
     default DataTrait computeDataTrait() {
         if (children().size() == 1) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java
index 5a791def8ee6f6d..96d34ff41973b55 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java
@@ -25,6 +25,7 @@
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation;
 import org.apache.doris.nereids.trees.plans.ObjectId;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.PlanType;
@@ -44,7 +45,7 @@
  * use for defer materialize top n
  */
 public class LogicalDeferMaterializeTopN<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_TYPE>
-        implements TopN {
+        implements TopN, BlockFuncDepsPropagation {
 
     private final LogicalTopN<? extends Plan> logicalTopN;
 
@@ -114,24 +115,6 @@ public List<Slot> computeOutput() {
                 .collect(ImmutableList.toImmutableList());
     }
 
-    @Override
-    public void computeUnique(DataTrait.Builder builder) {
-        if (getLimit() == 1) {
-            getOutput().forEach(builder::addUniqueSlot);
-        } else {
-            builder.addUniqueSlot(child(0).getLogicalProperties().getTrait());
-        }
-    }
-
-    @Override
-    public void computeUniform(DataTrait.Builder builder) {
-        if (getLimit() == 1) {
-            getOutput().forEach(builder::addUniformSlot);
-        } else {
-            builder.addUniformSlot(child(0).getLogicalProperties().getTrait());
-        }
-    }
-
     @Override
     public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
         return visitor.visitLogicalDeferMaterializeTopN(this, context);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java
index 3f93b97a6240ab0..65b3bf801d57d09 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java
@@ -104,14 +104,11 @@ public LogicalExcept withNewOutputs(List<NamedExpression> newOutputs) {
                 Optional.empty(), Optional.empty(), children);
     }
 
-    Map<Slot, Slot> constructReplaceMapForChild(int index) {
+    private Map<Slot, Slot> constructReplaceMapForChild(int index) {
         Map<Slot, Slot> replaceMap = new HashMap<>();
         List<Slot> output = getOutput();
-        List<? extends Slot> originalOutputs = regularChildrenOutputs.isEmpty()
-                ? child(index).getOutput()
-                : regularChildrenOutputs.get(index);
         for (int i = 0; i < output.size(); i++) {
-            replaceMap.put(originalOutputs.get(i), output.get(i));
+            replaceMap.put(regularChildrenOutputs.get(index).get(i), output.get(i));
         }
         return replaceMap;
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPlan.java
index b419c5718b05b0c..93031c7d15069aa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPlan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPlan.java
@@ -18,16 +18,11 @@
 package org.apache.doris.nereids.trees.plans.logical;
 
 import org.apache.doris.analysis.StmtType;
-import org.apache.doris.nereids.properties.DataTrait;
-import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.plans.Plan;
 
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Sets;
 
 import java.util.Optional;
-import java.util.Set;
 import java.util.function.BiFunction;
 import java.util.function.Supplier;
 
@@ -56,54 +51,6 @@ default LogicalPlan recomputeLogicalProperties() {
         return (LogicalPlan) withChildren(ImmutableList.copyOf(children()));
     }
 
-    /**
-     * Compute DataTrait for different plan
-     * Note: Unless you really know what you're doing, please use the following interface.
-     *   - BlockFDPropagation: clean the fd
-     *   - PropagateFD: propagate the fd
-     */
-    default DataTrait computeDataTrait() {
-        DataTrait.Builder fdBuilder = new DataTrait.Builder();
-        computeUniform(fdBuilder);
-        computeUnique(fdBuilder);
-        computeEqualSet(fdBuilder);
-        computeFd(fdBuilder);
-
-        for (Slot slot : getOutput()) {
-            Set<Slot> o = ImmutableSet.of(slot);
-            // all slots dependent unique slot
-            for (Set<Slot> uniqueSlot : fdBuilder.getAllUniqueAndNotNull()) {
-                fdBuilder.addDeps(uniqueSlot, o);
-            }
-            // uniform slot dependents all slots
-            for (Set<Slot> uniformSlot : fdBuilder.getAllUniformAndNotNull()) {
-                fdBuilder.addDeps(o, uniformSlot);
-            }
-        }
-        for (Set<Slot> equalSet : fdBuilder.calEqualSetList()) {
-            Set<Slot> validEqualSet = Sets.intersection(getOutputSet(), equalSet);
-            fdBuilder.addDepsByEqualSet(validEqualSet);
-            fdBuilder.addUniformByEqualSet(validEqualSet);
-            fdBuilder.addUniqueByEqualSet(validEqualSet);
-        }
-        Set<Slot> output = this.getOutputSet();
-        for (Plan child : children()) {
-            if (!output.containsAll(child.getOutputSet())) {
-                fdBuilder.pruneSlots(output);
-                break;
-            }
-        }
-        return fdBuilder.build();
-    }
-
-    void computeUnique(DataTrait.Builder builder);
-
-    void computeUniform(DataTrait.Builder builder);
-
-    void computeEqualSet(DataTrait.Builder builder);
-
-    void computeFd(DataTrait.Builder builder);
-
     default StmtType stmtType() {
         return StmtType.OTHER;
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSink.java
index ea2fa49e4664bbe..200d0cc71cf7004 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSink.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSink.java
@@ -18,6 +18,7 @@
 package org.apache.doris.nereids.trees.plans.logical;
 
 import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DataTrait;
 import org.apache.doris.nereids.properties.LogicalProperties;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.NamedExpression;
@@ -82,4 +83,24 @@ public boolean equals(Object o) {
     public int hashCode() {
         return Objects.hash(super.hashCode(), outputExprs);
     }
+
+    @Override
+    public void computeUnique(DataTrait.Builder builder) {
+        // should not be invoked
+    }
+
+    @Override
+    public void computeUniform(DataTrait.Builder builder) {
+        // should not be invoked
+    }
+
+    @Override
+    public void computeEqualSet(DataTrait.Builder builder) {
+        // should not be invoked
+    }
+
+    @Override
+    public void computeFd(DataTrait.Builder builder) {
+        // should not be invoked
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalAssertNumRows.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalAssertNumRows.java
index ba03050c7afa444..2636156111a9e9a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalAssertNumRows.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalAssertNumRows.java
@@ -18,6 +18,7 @@
 package org.apache.doris.nereids.trees.plans.physical;
 
 import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DataTrait;
 import org.apache.doris.nereids.properties.LogicalProperties;
 import org.apache.doris.nereids.properties.PhysicalProperties;
 import org.apache.doris.nereids.trees.expressions.AssertNumRowsElement;
@@ -134,4 +135,34 @@ public PhysicalAssertNumRows<CHILD_TYPE> resetLogicalProperties() {
         return new PhysicalAssertNumRows<>(assertNumRowsElement, groupExpression,
                 null, physicalProperties, statistics, child());
     }
+
+    @Override
+    public void computeUnique(DataTrait.Builder builder) {
+        if (assertNumRowsElement.getDesiredNumOfRows() == 1
+                && (assertNumRowsElement.getAssertion() == AssertNumRowsElement.Assertion.EQ
+                || assertNumRowsElement.getAssertion() == AssertNumRowsElement.Assertion.LT
+                || assertNumRowsElement.getAssertion() == AssertNumRowsElement.Assertion.LE)) {
+            getOutput().forEach(builder::addUniqueSlot);
+        }
+    }
+
+    @Override
+    public void computeUniform(DataTrait.Builder builder) {
+        if (assertNumRowsElement.getDesiredNumOfRows() == 1
+                && (assertNumRowsElement.getAssertion() == AssertNumRowsElement.Assertion.EQ
+                || assertNumRowsElement.getAssertion() == AssertNumRowsElement.Assertion.LT
+                || assertNumRowsElement.getAssertion() == AssertNumRowsElement.Assertion.LE)) {
+            getOutput().forEach(builder::addUniformSlot);
+        }
+    }
+
+    @Override
+    public void computeEqualSet(DataTrait.Builder builder) {
+        builder.addEqualSet(child().getLogicalProperties().getTrait());
+    }
+
+    @Override
+    public void computeFd(DataTrait.Builder builder) {
+        builder.addFuncDepsDG(child().getLogicalProperties().getTrait());
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEAnchor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEAnchor.java
index d3659e7177192fe..21a77d66597c68d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEAnchor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEAnchor.java
@@ -25,6 +25,7 @@
 import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.PropagateFuncDeps;
 import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
 import org.apache.doris.nereids.util.Utils;
 import org.apache.doris.statistics.Statistics;
@@ -42,7 +43,8 @@
 public class PhysicalCTEAnchor<
         LEFT_CHILD_TYPE extends Plan,
         RIGHT_CHILD_TYPE extends Plan>
-        extends PhysicalBinary<LEFT_CHILD_TYPE, RIGHT_CHILD_TYPE> {
+        extends PhysicalBinary<LEFT_CHILD_TYPE, RIGHT_CHILD_TYPE> implements
+        PropagateFuncDeps {
     private final CTEId cteId;
 
     public PhysicalCTEAnchor(CTEId cteId, LogicalProperties logicalProperties,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEConsumer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEConsumer.java
index 385c411e7217de2..720e1eee0b5087d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEConsumer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEConsumer.java
@@ -22,6 +22,7 @@
 import org.apache.doris.nereids.properties.PhysicalProperties;
 import org.apache.doris.nereids.trees.expressions.CTEId;
 import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.PlanType;
 import org.apache.doris.nereids.trees.plans.RelationId;
@@ -42,7 +43,7 @@
 /**
  * Physical CTE consumer.
  */
-public class PhysicalCTEConsumer extends PhysicalRelation {
+public class PhysicalCTEConsumer extends PhysicalRelation implements BlockFuncDepsPropagation {
 
     private final CTEId cteId;
     private final Multimap<Slot, Slot> producerToConsumerSlotMap;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEProducer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEProducer.java
index 568b8e6660ab396..d06ba006d051aa2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEProducer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEProducer.java
@@ -25,6 +25,7 @@
 import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.PropagateFuncDeps;
 import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
 import org.apache.doris.nereids.util.Utils;
 import org.apache.doris.statistics.Statistics;
@@ -39,7 +40,8 @@
 /**
  * Physical CTE producer.
  */
-public class PhysicalCTEProducer<CHILD_TYPE extends Plan> extends PhysicalUnary<CHILD_TYPE> {
+public class PhysicalCTEProducer<CHILD_TYPE extends Plan> extends PhysicalUnary<CHILD_TYPE>
+        implements PropagateFuncDeps {
 
     private final CTEId cteId;
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCatalogRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCatalogRelation.java
index 17977fe054a664c..4a4b6b61ffd961d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCatalogRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCatalogRelation.java
@@ -17,12 +17,16 @@
 
 package org.apache.doris.nereids.trees.plans.physical;
 
+import org.apache.doris.catalog.Column;
 import org.apache.doris.catalog.DatabaseIf;
 import org.apache.doris.catalog.Env;
 import org.apache.doris.catalog.TableIf;
+import org.apache.doris.catalog.constraint.PrimaryKeyConstraint;
+import org.apache.doris.catalog.constraint.UniqueConstraint;
 import org.apache.doris.datasource.CatalogIf;
 import org.apache.doris.nereids.exceptions.AnalysisException;
 import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DataTrait;
 import org.apache.doris.nereids.properties.LogicalProperties;
 import org.apache.doris.nereids.properties.PhysicalProperties;
 import org.apache.doris.nereids.trees.expressions.Slot;
@@ -36,10 +40,12 @@
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.Set;
 
 /**
  * relation generated from TableIf
@@ -148,4 +154,47 @@ public String shapeInfo() {
         }
         return shapeBuilder.toString();
     }
+
+    @Override
+    public void computeUnique(DataTrait.Builder builder) {
+        Set<Slot> outputSet = Utils.fastToImmutableSet(getOutputSet());
+        for (PrimaryKeyConstraint c : table.getPrimaryKeyConstraints()) {
+            Set<Column> columns = c.getPrimaryKeys(table);
+            builder.addUniqueSlot((ImmutableSet) findSlotsByColumn(outputSet, columns));
+        }
+
+        for (UniqueConstraint c : table.getUniqueConstraints()) {
+            Set<Column> columns = c.getUniqueKeys(table);
+            builder.addUniqueSlot((ImmutableSet) findSlotsByColumn(outputSet, columns));
+        }
+    }
+
+    @Override
+    public void computeUniform(DataTrait.Builder builder) {
+        // No uniform slot for catalog relation
+    }
+
+    private ImmutableSet<SlotReference> findSlotsByColumn(Set<Slot> outputSet, Set<Column> columns) {
+        ImmutableSet.Builder<SlotReference> slotSet = ImmutableSet.builderWithExpectedSize(columns.size());
+        for (Slot slot : outputSet) {
+            if (!(slot instanceof SlotReference)) {
+                continue;
+            }
+            SlotReference slotRef = (SlotReference) slot;
+            if (slotRef.getColumn().isPresent() && columns.contains(slotRef.getColumn().get())) {
+                slotSet.add(slotRef);
+            }
+        }
+        return slotSet.build();
+    }
+
+    @Override
+    public void computeEqualSet(DataTrait.Builder builder) {
+        // don't generate any equal pair
+    }
+
+    @Override
+    public void computeFd(DataTrait.Builder builder) {
+        // don't generate any equal pair
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDeferMaterializeTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDeferMaterializeTopN.java
index 74088498d723582..14412f2cfc62df3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDeferMaterializeTopN.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDeferMaterializeTopN.java
@@ -18,6 +18,7 @@
 package org.apache.doris.nereids.trees.plans.physical;
 
 import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DataTrait;
 import org.apache.doris.nereids.properties.LogicalProperties;
 import org.apache.doris.nereids.properties.PhysicalProperties;
 import org.apache.doris.nereids.trees.expressions.ExprId;
@@ -177,6 +178,34 @@ public String toString() {
         );
     }
 
+    @Override
+    public void computeUnique(DataTrait.Builder builder) {
+        if (getLimit() == 1) {
+            getOutput().forEach(builder::addUniqueSlot);
+        } else {
+            builder.addUniqueSlot(child(0).getLogicalProperties().getTrait());
+        }
+    }
+
+    @Override
+    public void computeUniform(DataTrait.Builder builder) {
+        if (getLimit() == 1) {
+            getOutput().forEach(builder::addUniformSlot);
+        } else {
+            builder.addUniformSlot(child(0).getLogicalProperties().getTrait());
+        }
+    }
+
+    @Override
+    public void computeFd(DataTrait.Builder builder) {
+        builder.addFuncDepsDG(child().getLogicalProperties().getTrait());
+    }
+
+    @Override
+    public void computeEqualSet(DataTrait.Builder builder) {
+        builder.addEqualSet(child().getLogicalProperties().getTrait());
+    }
+
     @Override
     public ObjectId getObjectId() {
         return id;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDistribute.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDistribute.java
index d85148af38e9d05..7bd238c75228d9a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDistribute.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDistribute.java
@@ -25,6 +25,7 @@
 import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.PropagateFuncDeps;
 import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
 import org.apache.doris.nereids.util.Utils;
 import org.apache.doris.statistics.Statistics;
@@ -39,7 +40,8 @@
 /**
  * Enforcer plan.
  */
-public class PhysicalDistribute<CHILD_TYPE extends Plan> extends PhysicalUnary<CHILD_TYPE> {
+public class PhysicalDistribute<CHILD_TYPE extends Plan> extends PhysicalUnary<CHILD_TYPE>
+        implements PropagateFuncDeps {
 
     protected DistributionSpec distributionSpec;
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEmptyRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEmptyRelation.java
index 30beb0d2f41aaec..b1b5ea2beeecd52 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEmptyRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEmptyRelation.java
@@ -27,6 +27,7 @@
 import org.apache.doris.nereids.properties.PhysicalProperties;
 import org.apache.doris.nereids.trees.expressions.NamedExpression;
 import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation;
 import org.apache.doris.nereids.trees.plans.ComputeResultSet;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.PlanType;
@@ -52,7 +53,8 @@
  * e.g.
  * select * from tbl limit 0
  */
-public class PhysicalEmptyRelation extends PhysicalRelation implements EmptyRelation, ComputeResultSet {
+public class PhysicalEmptyRelation extends PhysicalRelation
+        implements EmptyRelation, ComputeResultSet, BlockFuncDepsPropagation {
 
     private final List<? extends NamedExpression> projects;
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalExcept.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalExcept.java
index acfd977856f145e..4977239030e28ed 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalExcept.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalExcept.java
@@ -18,9 +18,11 @@
 package org.apache.doris.nereids.trees.plans.physical;
 
 import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DataTrait;
 import org.apache.doris.nereids.properties.LogicalProperties;
 import org.apache.doris.nereids.properties.PhysicalProperties;
 import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.SlotReference;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.PlanType;
@@ -28,7 +30,11 @@
 import org.apache.doris.nereids.util.Utils;
 import org.apache.doris.statistics.Statistics;
 
+import com.google.common.collect.ImmutableSet;
+
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 
 /**
@@ -109,4 +115,41 @@ public PhysicalExcept resetLogicalProperties() {
         return new PhysicalExcept(qualifier, outputs, regularChildrenOutputs, Optional.empty(),
                 getLogicalProperties(), physicalProperties, statistics, children);
     }
+
+    private Map<Slot, Slot> constructReplaceMapForChild(int index) {
+        Map<Slot, Slot> replaceMap = new HashMap<>();
+        List<Slot> output = getOutput();
+        for (int i = 0; i < output.size(); i++) {
+            replaceMap.put(regularChildrenOutputs.get(index).get(i), output.get(i));
+        }
+        return replaceMap;
+    }
+
+    @Override
+    public void computeUnique(DataTrait.Builder builder) {
+        builder.addUniqueSlot(child(0).getLogicalProperties().getTrait());
+        if (qualifier == Qualifier.DISTINCT) {
+            builder.addUniqueSlot(ImmutableSet.copyOf(getOutput()));
+        }
+        builder.replaceUniqueBy(constructReplaceMapForChild(0));
+    }
+
+    @Override
+    public void computeEqualSet(DataTrait.Builder builder) {
+        builder.addEqualSet(child(0).getLogicalProperties().getTrait());
+        builder.replaceEqualSetBy(constructReplaceMapForChild(0));
+    }
+
+    @Override
+    public void computeFd(DataTrait.Builder builder) {
+        builder.addFuncDepsDG(child(0).getLogicalProperties().getTrait());
+        builder.replaceFuncDepsBy(constructReplaceMapForChild(0));
+    }
+
+    @Override
+    public void computeUniform(DataTrait.Builder builder) {
+        builder.addUniformSlot(child(0).getLogicalProperties().getTrait());
+        builder.replaceUniformBy(constructReplaceMapForChild(0));
+    }
+
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFilter.java
index 42327e7b92541e0..8a2390e4bad4280 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFilter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFilter.java
@@ -17,7 +17,9 @@
 
 package org.apache.doris.nereids.trees.plans.physical;
 
+import org.apache.doris.common.Pair;
 import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DataTrait;
 import org.apache.doris.nereids.properties.LogicalProperties;
 import org.apache.doris.nereids.properties.PhysicalProperties;
 import org.apache.doris.nereids.trees.expressions.Expression;
@@ -26,6 +28,7 @@
 import org.apache.doris.nereids.trees.plans.PlanType;
 import org.apache.doris.nereids.trees.plans.algebra.Filter;
 import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.ExpressionUtils;
 import org.apache.doris.nereids.util.Utils;
 import org.apache.doris.statistics.Statistics;
 
@@ -34,6 +37,7 @@
 import com.google.common.collect.ImmutableSet;
 
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
@@ -158,4 +162,34 @@ public PhysicalFilter<Plan> resetLogicalProperties() {
         return new PhysicalFilter<>(conjuncts, groupExpression, null, physicalProperties,
                 statistics, child());
     }
+
+    @Override
+    public void computeUnique(DataTrait.Builder builder) {
+        builder.addUniqueSlot(child(0).getLogicalProperties().getTrait());
+    }
+
+    @Override
+    public void computeUniform(DataTrait.Builder builder) {
+        for (Expression e : getConjuncts()) {
+            Map<Slot, Expression> uniformSlots = ExpressionUtils.extractUniformSlot(e);
+            for (Map.Entry<Slot, Expression> entry : uniformSlots.entrySet()) {
+                builder.addUniformSlotAndLiteral(entry.getKey(), entry.getValue());
+            }
+        }
+        builder.addUniformSlot(child(0).getLogicalProperties().getTrait());
+    }
+
+    @Override
+    public void computeEqualSet(DataTrait.Builder builder) {
+        builder.addEqualSet(child().getLogicalProperties().getTrait());
+        for (Expression expression : getConjuncts()) {
+            Optional<Pair<Slot, Slot>> equalSlot = ExpressionUtils.extractEqualSlot(expression);
+            equalSlot.ifPresent(slotSlotPair -> builder.addEqualPair(slotSlotPair.first, slotSlotPair.second));
+        }
+    }
+
+    @Override
+    public void computeFd(DataTrait.Builder builder) {
+        builder.addFuncDepsDG(child().getLogicalProperties().getTrait());
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalGenerate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalGenerate.java
index 5cd8bd238c295f4..8d6dd5c5c83fc9c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalGenerate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalGenerate.java
@@ -18,6 +18,7 @@
 package org.apache.doris.nereids.trees.plans.physical;
 
 import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DataTrait;
 import org.apache.doris.nereids.properties.LogicalProperties;
 import org.apache.doris.nereids.properties.PhysicalProperties;
 import org.apache.doris.nereids.trees.expressions.Expression;
@@ -164,4 +165,24 @@ public PhysicalGenerate<CHILD_TYPE> resetLogicalProperties() {
                 Optional.empty(), null, physicalProperties,
                 statistics, child());
     }
+
+    @Override
+    public void computeUnique(DataTrait.Builder builder) {
+        // don't generate and propagate unique
+    }
+
+    @Override
+    public void computeUniform(DataTrait.Builder builder) {
+        builder.addUniformSlot(child(0).getLogicalProperties().getTrait());
+    }
+
+    @Override
+    public void computeEqualSet(DataTrait.Builder builder) {
+        builder.addEqualSet(child().getLogicalProperties().getTrait());
+    }
+
+    @Override
+    public void computeFd(DataTrait.Builder builder) {
+        builder.addFuncDepsDG(child().getLogicalProperties().getTrait());
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java
index 94e86506b587513..34f136243cf861b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java
@@ -18,6 +18,7 @@
 package org.apache.doris.nereids.trees.plans.physical;
 
 import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DataTrait;
 import org.apache.doris.nereids.properties.LogicalProperties;
 import org.apache.doris.nereids.properties.OrderKey;
 import org.apache.doris.nereids.properties.PhysicalProperties;
@@ -27,8 +28,11 @@
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.NamedExpression;
 import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.VirtualSlotReference;
 import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
 import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateParam;
+import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
+import org.apache.doris.nereids.trees.expressions.functions.agg.Ndv;
 import org.apache.doris.nereids.trees.expressions.functions.agg.NullableAggregateFunction;
 import org.apache.doris.nereids.trees.plans.AggMode;
 import org.apache.doris.nereids.trees.plans.AggPhase;
@@ -43,6 +47,7 @@
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 
 import java.util.List;
 import java.util.Objects;
@@ -381,4 +386,91 @@ private List<NamedExpression> adjustNullableForOutputs(List<NamedExpression> out
             return output;
         });
     }
+
+    private boolean isUniqueGroupByUnique(NamedExpression namedExpression) {
+        if (namedExpression.children().size() != 1) {
+            return false;
+        }
+        Expression agg = namedExpression.child(0);
+        return ExpressionUtils.isInjectiveAgg(agg)
+                && child().getLogicalProperties().getTrait().isUniqueAndNotNull(agg.getInputSlots());
+    }
+
+    private boolean isUniformGroupByUnique(NamedExpression namedExpression) {
+        if (namedExpression.children().size() != 1) {
+            return false;
+        }
+        Expression agg = namedExpression.child(0);
+        return agg instanceof Count || agg instanceof Ndv;
+    }
+
+    @Override
+    public void computeUnique(DataTrait.Builder builder) {
+        if (groupByExpressions.stream().anyMatch(s -> s instanceof VirtualSlotReference)) {
+            // roll up may generate new data
+            return;
+        }
+        DataTrait childFd = child(0).getLogicalProperties().getTrait();
+        ImmutableSet<Slot> groupByKeys = groupByExpressions.stream()
+                .map(s -> (Slot) s)
+                .collect(ImmutableSet.toImmutableSet());
+        // when group by all tuples, the result only have one row
+        if (groupByExpressions.isEmpty() || childFd.isUniformAndNotNull(groupByKeys)) {
+            getOutput().forEach(builder::addUniqueSlot);
+            return;
+        }
+
+        // propagate all unique slots
+        builder.addUniqueSlot(childFd);
+
+        // group by keys is unique
+        builder.addUniqueSlot(groupByKeys);
+
+        // group by unique may has unique aggregate result
+        if (childFd.isUniqueAndNotNull(groupByKeys)) {
+            for (NamedExpression namedExpression : getOutputExpressions()) {
+                if (isUniqueGroupByUnique(namedExpression)) {
+                    builder.addUniqueSlot(namedExpression.toSlot());
+                }
+            }
+        }
+    }
+
+    @Override
+    public void computeUniform(DataTrait.Builder builder) {
+        // always propagate uniform
+        DataTrait childFd = child(0).getLogicalProperties().getTrait();
+        builder.addUniformSlot(childFd);
+
+        if (groupByExpressions.stream().anyMatch(s -> s instanceof VirtualSlotReference)) {
+            // roll up may generate new data
+            return;
+        }
+        ImmutableSet<Slot> groupByKeys = groupByExpressions.stream()
+                .map(s -> (Slot) s)
+                .collect(ImmutableSet.toImmutableSet());
+        // when group by all tuples, the result only have one row
+        if (groupByExpressions.isEmpty() || childFd.isUniformAndNotNull(groupByKeys)) {
+            getOutput().forEach(builder::addUniformSlot);
+            return;
+        }
+
+        if (childFd.isUniqueAndNotNull(groupByKeys)) {
+            for (NamedExpression namedExpression : getOutputExpressions()) {
+                if (isUniformGroupByUnique(namedExpression)) {
+                    builder.addUniformSlot(namedExpression.toSlot());
+                }
+            }
+        }
+    }
+
+    @Override
+    public void computeEqualSet(DataTrait.Builder builder) {
+        builder.addEqualSet(child().getLogicalProperties().getTrait());
+    }
+
+    @Override
+    public void computeFd(DataTrait.Builder builder) {
+        builder.addFuncDepsDG(child().getLogicalProperties().getTrait());
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java
index ccfb3f119372e61..e4caab5bce36772 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java
@@ -20,11 +20,14 @@
 import org.apache.doris.common.Pair;
 import org.apache.doris.nereids.hint.DistributeHint;
 import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DataTrait;
 import org.apache.doris.nereids.properties.LogicalProperties;
 import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.trees.expressions.EqualTo;
 import org.apache.doris.nereids.trees.expressions.ExprId;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference;
+import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.plans.JoinType;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.PlanType;
@@ -37,10 +40,12 @@
 import com.google.common.base.Preconditions;
 
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
+import javax.annotation.Nullable;
 
 /**
  * Physical hash join plan.
@@ -224,4 +229,136 @@ public PhysicalHashJoin<LEFT_CHILD_TYPE, RIGHT_CHILD_TYPE> resetLogicalPropertie
                 markJoinConjuncts, hint, markJoinSlotReference, groupExpression, null,
                 physicalProperties, statistics, left(), right());
     }
+
+    private @Nullable Pair<Set<Slot>, Set<Slot>> extractNullRejectHashKeys() {
+        // this function is only used by computeFuncDeps, and function dependence calculation is disabled for mark join
+        // so markJoinConjuncts is not processed now
+        Set<Slot> leftKeys = new HashSet<>();
+        Set<Slot> rightKeys = new HashSet<>();
+        for (Expression expression : hashJoinConjuncts) {
+            // Note we don't support null-safe predicate right now, because we just check uniqueness for join keys
+            if (!(expression instanceof EqualTo
+                    && ((EqualTo) expression).left() instanceof Slot
+                    && ((EqualTo) expression).right() instanceof Slot)) {
+                return null;
+            }
+            Slot leftKey = (Slot) ((EqualTo) expression).left();
+            Slot rightKey = (Slot) ((EqualTo) expression).right();
+            if (left().getOutputSet().contains(leftKey)) {
+                leftKeys.add(leftKey);
+                rightKeys.add(rightKey);
+            } else {
+                leftKeys.add(rightKey);
+                rightKeys.add(leftKey);
+            }
+        }
+        return Pair.of(leftKeys, rightKeys);
+    }
+
+    @Override
+    public void computeUnique(DataTrait.Builder builder) {
+        if (isMarkJoin()) {
+            // TODO disable function dependence calculation for mark join, but need re-think this in future.
+            return;
+        }
+        if (joinType.isLeftSemiOrAntiJoin()) {
+            builder.addUniqueSlot(left().getLogicalProperties().getTrait());
+        } else if (joinType.isRightSemiOrAntiJoin()) {
+            builder.addUniqueSlot(right().getLogicalProperties().getTrait());
+        }
+        // if there is non-equal join conditions, don't propagate unique
+        if (hashJoinConjuncts.isEmpty()) {
+            return;
+        }
+        Pair<Set<Slot>, Set<Slot>> keys = extractNullRejectHashKeys();
+        if (keys == null) {
+            return;
+        }
+
+        // Note here we only check whether the left is unique.
+        // So the hash condition can't be null-safe
+        // TODO: consider Null-safe hash condition when left and rigth is not nullable
+        boolean isLeftUnique = left().getLogicalProperties()
+                .getTrait().isUnique(keys.first);
+        boolean isRightUnique = right().getLogicalProperties()
+                .getTrait().isUnique(keys.second);
+
+        // left/right outer join propagate left/right uniforms slots
+        // And if the right/left hash keys is unique,
+        // join can propagate left/right functional dependencies
+        if (joinType.isLeftOuterJoin() && isRightUnique) {
+            builder.addUniqueSlot(left().getLogicalProperties().getTrait());
+        } else if (joinType.isRightOuterJoin() && isLeftUnique) {
+            builder.addUniqueSlot(right().getLogicalProperties().getTrait());
+        } else if (joinType.isInnerJoin() && isLeftUnique && isRightUnique) {
+            // inner join propagate uniforms slots
+            // And if the hash keys is unique, inner join can propagate all functional dependencies
+            builder.addDataTrait(left().getLogicalProperties().getTrait());
+            builder.addDataTrait(right().getLogicalProperties().getTrait());
+        }
+    }
+
+    @Override
+    public void computeUniform(DataTrait.Builder builder) {
+        if (isMarkJoin()) {
+            // TODO disable function dependence calculation for mark join, but need re-think this in future.
+            return;
+        }
+        switch (joinType) {
+            case INNER_JOIN:
+            case CROSS_JOIN:
+                builder.addUniformSlot(left().getLogicalProperties().getTrait());
+                builder.addUniformSlot(right().getLogicalProperties().getTrait());
+                break;
+            case LEFT_SEMI_JOIN:
+            case LEFT_ANTI_JOIN:
+            case NULL_AWARE_LEFT_ANTI_JOIN:
+                builder.addUniformSlot(left().getLogicalProperties().getTrait());
+                break;
+            case RIGHT_SEMI_JOIN:
+            case RIGHT_ANTI_JOIN:
+                builder.addUniformSlot(right().getLogicalProperties().getTrait());
+                break;
+            case LEFT_OUTER_JOIN:
+                builder.addUniformSlot(left().getLogicalProperties().getTrait());
+                builder.addUniformSlotForOuterJoinNullableSide(right().getLogicalProperties().getTrait());
+                break;
+            case RIGHT_OUTER_JOIN:
+                builder.addUniformSlot(right().getLogicalProperties().getTrait());
+                builder.addUniformSlotForOuterJoinNullableSide(left().getLogicalProperties().getTrait());
+                break;
+            case FULL_OUTER_JOIN:
+                builder.addUniformSlotForOuterJoinNullableSide(left().getLogicalProperties().getTrait());
+                builder.addUniformSlotForOuterJoinNullableSide(right().getLogicalProperties().getTrait());
+                break;
+            default:
+                break;
+        }
+    }
+
+    @Override
+    public void computeEqualSet(DataTrait.Builder builder) {
+        if (!joinType.isLeftSemiOrAntiJoin()) {
+            builder.addEqualSet(right().getLogicalProperties().getTrait());
+        }
+        if (!joinType.isRightSemiOrAntiJoin()) {
+            builder.addEqualSet(left().getLogicalProperties().getTrait());
+        }
+        if (joinType.isInnerJoin()) {
+            for (Expression expression : getHashJoinConjuncts()) {
+                Optional<Pair<Slot, Slot>> equalSlot = ExpressionUtils.extractEqualSlot(expression);
+                equalSlot.ifPresent(slotSlotPair -> builder.addEqualPair(slotSlotPair.first, slotSlotPair.second));
+            }
+        }
+    }
+
+    @Override
+    public void computeFd(DataTrait.Builder builder) {
+        if (!joinType.isLeftSemiOrAntiJoin()) {
+            builder.addFuncDepsDG(right().getLogicalProperties().getTrait());
+        }
+        if (!joinType.isRightSemiOrAntiJoin()) {
+            builder.addFuncDepsDG(left().getLogicalProperties().getTrait());
+        }
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIntersect.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIntersect.java
index 0919868b2dd4972..3194939a9e64812 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIntersect.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIntersect.java
@@ -18,9 +18,11 @@
 package org.apache.doris.nereids.trees.plans.physical;
 
 import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DataTrait;
 import org.apache.doris.nereids.properties.LogicalProperties;
 import org.apache.doris.nereids.properties.PhysicalProperties;
 import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.SlotReference;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.PlanType;
@@ -28,7 +30,11 @@
 import org.apache.doris.nereids.util.Utils;
 import org.apache.doris.statistics.Statistics;
 
+import com.google.common.collect.ImmutableSet;
+
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 
 /**
@@ -108,4 +114,56 @@ public PhysicalIntersect resetLogicalProperties() {
         return new PhysicalIntersect(qualifier, outputs, regularChildrenOutputs,
                 Optional.empty(), null, physicalProperties, statistics, children);
     }
+
+    Map<Slot, Slot> constructReplaceMap() {
+        Map<Slot, Slot> replaceMap = new HashMap<>();
+        for (int i = 0; i < children.size(); i++) {
+            List<? extends Slot> originOutputs = this.regularChildrenOutputs.size() == children.size()
+                    ? child(i).getOutput()
+                    : regularChildrenOutputs.get(i);
+            for (int j = 0; j < originOutputs.size(); j++) {
+                replaceMap.put(originOutputs.get(j), getOutput().get(j));
+            }
+        }
+        return replaceMap;
+    }
+
+    @Override
+    public void computeUnique(DataTrait.Builder builder) {
+        for (Plan child : children) {
+            builder.addUniqueSlot(
+                    child.getLogicalProperties().getTrait());
+        }
+        builder.replaceUniqueBy(constructReplaceMap());
+        if (qualifier == Qualifier.DISTINCT) {
+            builder.addUniqueSlot(ImmutableSet.copyOf(getOutput()));
+        }
+    }
+
+    @Override
+    public void computeUniform(DataTrait.Builder builder) {
+        for (Plan child : children) {
+            builder.addUniformSlot(
+                    child.getLogicalProperties().getTrait());
+        }
+        builder.replaceUniformBy(constructReplaceMap());
+    }
+
+    @Override
+    public void computeEqualSet(DataTrait.Builder builder) {
+        for (Plan child : children) {
+            builder.addEqualSet(
+                    child.getLogicalProperties().getTrait());
+        }
+        builder.replaceEqualSetBy(constructReplaceMap());
+    }
+
+    @Override
+    public void computeFd(DataTrait.Builder builder) {
+        for (Plan child : children) {
+            builder.addFuncDepsDG(
+                    child.getLogicalProperties().getTrait());
+        }
+        builder.replaceFuncDepsBy(constructReplaceMap());
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLimit.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLimit.java
index bbc2143df61bfb5..27155ae57f7e5e8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLimit.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLimit.java
@@ -18,6 +18,7 @@
 package org.apache.doris.nereids.trees.plans.physical;
 
 import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DataTrait;
 import org.apache.doris.nereids.properties.LogicalProperties;
 import org.apache.doris.nereids.properties.PhysicalProperties;
 import org.apache.doris.nereids.trees.expressions.Expression;
@@ -184,4 +185,32 @@ public PhysicalLimit<CHILD_TYPE> resetLogicalProperties() {
         return new PhysicalLimit<>(limit, offset, phase, groupExpression, null, physicalProperties,
                 statistics, child());
     }
+
+    @Override
+    public void computeUnique(DataTrait.Builder builder) {
+        if (getLimit() == 1) {
+            getOutput().forEach(builder::addUniqueSlot);
+        } else {
+            builder.addUniqueSlot(child(0).getLogicalProperties().getTrait());
+        }
+    }
+
+    @Override
+    public void computeUniform(DataTrait.Builder builder) {
+        if (getLimit() == 1) {
+            getOutput().forEach(builder::addUniformSlot);
+        } else {
+            builder.addUniformSlot(child(0).getLogicalProperties().getTrait());
+        }
+    }
+
+    @Override
+    public void computeEqualSet(DataTrait.Builder builder) {
+        builder.addEqualSet(child().getLogicalProperties().getTrait());
+    }
+
+    @Override
+    public void computeFd(DataTrait.Builder builder) {
+        builder.addFuncDepsDG(child().getLogicalProperties().getTrait());
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalNestedLoopJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalNestedLoopJoin.java
index b8e5c17e0342bbe..609ebefbeff31ed 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalNestedLoopJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalNestedLoopJoin.java
@@ -24,6 +24,7 @@
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference;
 import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation;
 import org.apache.doris.nereids.trees.plans.DistributeType;
 import org.apache.doris.nereids.trees.plans.JoinType;
 import org.apache.doris.nereids.trees.plans.Plan;
@@ -49,7 +50,8 @@
 public class PhysicalNestedLoopJoin<
         LEFT_CHILD_TYPE extends Plan,
         RIGHT_CHILD_TYPE extends Plan>
-        extends AbstractPhysicalJoin<LEFT_CHILD_TYPE, RIGHT_CHILD_TYPE> {
+        extends AbstractPhysicalJoin<LEFT_CHILD_TYPE, RIGHT_CHILD_TYPE>
+        implements BlockFuncDepsPropagation {
 
     /*
     bitmap_contains(...) or Not(bitmap_contains(...)) can be used as bitmap runtime filter condition
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOneRowRelation.java
index e3c1ca7f4935251..edde1c17c7a8add 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOneRowRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOneRowRelation.java
@@ -24,8 +24,10 @@
 import org.apache.doris.nereids.SqlCacheContext;
 import org.apache.doris.nereids.StatementContext;
 import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DataTrait;
 import org.apache.doris.nereids.properties.LogicalProperties;
 import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.trees.expressions.Alias;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.NamedExpression;
 import org.apache.doris.nereids.trees.expressions.Slot;
@@ -47,7 +49,9 @@
 import com.google.common.collect.Lists;
 
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 
@@ -172,4 +176,32 @@ public Optional<ResultSet> computeResultInFe(
         }
         return Optional.of(resultSet);
     }
+
+    @Override
+    public void computeUnique(DataTrait.Builder builder) {
+        getOutput().forEach(builder::addUniqueSlot);
+    }
+
+    @Override
+    public void computeUniform(DataTrait.Builder builder) {
+        getOutput().forEach(builder::addUniformSlot);
+    }
+
+    @Override
+    public void computeEqualSet(DataTrait.Builder builder) {
+        Map<Expression, NamedExpression> aliasMap = new HashMap<>();
+        for (NamedExpression namedExpr : projects) {
+            if (namedExpr instanceof Alias) {
+                if (aliasMap.containsKey(namedExpr.child(0))) {
+                    builder.addEqualPair(namedExpr.toSlot(), aliasMap.get(namedExpr.child(0)).toSlot());
+                }
+                aliasMap.put(namedExpr.child(0), namedExpr);
+            }
+        }
+    }
+
+    @Override
+    public void computeFd(DataTrait.Builder builder) {
+        // don't generate
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPartitionTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPartitionTopN.java
index 971042db420251e..21eef99135b04f1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPartitionTopN.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPartitionTopN.java
@@ -26,6 +26,7 @@
 import org.apache.doris.nereids.trees.plans.PartitionTopnPhase;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.PropagateFuncDeps;
 import org.apache.doris.nereids.trees.plans.WindowFuncType;
 import org.apache.doris.nereids.trees.plans.algebra.PartitionTopN;
 import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
@@ -43,7 +44,8 @@
 /**
  * Physical partition-top-N plan.
  */
-public class PhysicalPartitionTopN<CHILD_TYPE extends Plan> extends PhysicalUnary<CHILD_TYPE> implements PartitionTopN {
+public class PhysicalPartitionTopN<CHILD_TYPE extends Plan> extends PhysicalUnary<CHILD_TYPE>
+        implements PartitionTopN, PropagateFuncDeps {
     private final WindowFuncType function;
     private final List<Expression> partitionKeys;
     private final List<OrderKey> orderKeys;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalProject.java
index 0e7bdf36e8f7902..8dfb5cf55ddc6d3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalProject.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalProject.java
@@ -18,6 +18,7 @@
 package org.apache.doris.nereids.trees.plans.physical;
 
 import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DataTrait;
 import org.apache.doris.nereids.properties.LogicalProperties;
 import org.apache.doris.nereids.properties.PhysicalProperties;
 import org.apache.doris.nereids.trees.expressions.Add;
@@ -26,10 +27,12 @@
 import org.apache.doris.nereids.trees.expressions.NamedExpression;
 import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Uuid;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.PlanType;
 import org.apache.doris.nereids.trees.plans.algebra.Project;
 import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.ExpressionUtils;
 import org.apache.doris.nereids.util.Utils;
 import org.apache.doris.statistics.Statistics;
 
@@ -40,7 +43,9 @@
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
 
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
@@ -247,4 +252,71 @@ public List<List<NamedExpression>> getMultiLayerProjects() {
     public void setMultiLayerProjects(List<List<NamedExpression>> multiLayers) {
         this.multiLayerProjects = multiLayers;
     }
+
+    @Override
+    public void computeUnique(DataTrait.Builder builder) {
+        builder.addUniqueSlot(child(0).getLogicalProperties().getTrait());
+        for (NamedExpression proj : getProjects()) {
+            if (proj.children().isEmpty()) {
+                continue;
+            }
+            if (proj.child(0) instanceof Uuid) {
+                builder.addUniqueSlot(proj.toSlot());
+            } else if (ExpressionUtils.isInjective(proj.child(0))) {
+                ImmutableSet<Slot> inputs = ImmutableSet.copyOf(proj.getInputSlots());
+                if (child(0).getLogicalProperties().getTrait().isUnique(inputs)) {
+                    builder.addUniqueSlot(proj.toSlot());
+                }
+            }
+        }
+    }
+
+    @Override
+    public void computeUniform(DataTrait.Builder builder) {
+        builder.addUniformSlot(child(0).getLogicalProperties().getTrait());
+        for (NamedExpression proj : getProjects()) {
+            if (!(proj instanceof Alias)) {
+                continue;
+            }
+            if (proj.child(0).isConstant()) {
+                builder.addUniformSlotAndLiteral(proj.toSlot(), proj.child(0));
+            } else if (proj.child(0) instanceof Slot) {
+                Slot slot = (Slot) proj.child(0);
+                DataTrait childTrait = child(0).getLogicalProperties().getTrait();
+                if (childTrait.isUniformAndHasConstValue(slot)) {
+                    builder.addUniformSlotAndLiteral(proj.toSlot(),
+                            child(0).getLogicalProperties().getTrait().getUniformValue(slot).get());
+                } else if (childTrait.isUniform(slot)) {
+                    builder.addUniformSlot(proj.toSlot());
+                }
+            }
+        }
+    }
+
+    @Override
+    public void computeEqualSet(DataTrait.Builder builder) {
+        Map<Expression, NamedExpression> aliasMap = new HashMap<>();
+        builder.addEqualSet(child().getLogicalProperties().getTrait());
+        for (NamedExpression expr : getProjects()) {
+            if (expr instanceof Alias) {
+                if (aliasMap.containsKey(expr.child(0))) {
+                    builder.addEqualPair(expr.toSlot(), aliasMap.get(expr.child(0)).toSlot());
+                }
+                aliasMap.put(expr.child(0), expr);
+                if (expr.child(0).isSlot()) {
+                    builder.addEqualPair(expr.toSlot(), (Slot) expr.child(0));
+                }
+            }
+        }
+    }
+
+    @Override
+    public void computeFd(DataTrait.Builder builder) {
+        builder.addFuncDepsDG(child().getLogicalProperties().getTrait());
+        for (NamedExpression expr : getProjects()) {
+            if (!expr.isSlot()) {
+                builder.addDeps(expr.getInputSlots(), ImmutableSet.of(expr.toSlot()));
+            }
+        }
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalQuickSort.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalQuickSort.java
index 0e377b46d238ea2..804cf4b7c66d759 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalQuickSort.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalQuickSort.java
@@ -24,6 +24,7 @@
 import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.PropagateFuncDeps;
 import org.apache.doris.nereids.trees.plans.SortPhase;
 import org.apache.doris.nereids.trees.plans.algebra.Sort;
 import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
@@ -43,7 +44,8 @@
  *      GATHER_SORT: SORT the gather data
  *  MERGE_SORT AND GATHER_SORT need require gather for their child
  */
-public class PhysicalQuickSort<CHILD_TYPE extends Plan> extends AbstractPhysicalSort<CHILD_TYPE> implements Sort {
+public class PhysicalQuickSort<CHILD_TYPE extends Plan> extends AbstractPhysicalSort<CHILD_TYPE>
+        implements Sort, PropagateFuncDeps {
     public PhysicalQuickSort(List<OrderKey> orderKeys,
             SortPhase phase, LogicalProperties logicalProperties, CHILD_TYPE child) {
         this(orderKeys, phase, Optional.empty(), logicalProperties, child);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRepeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRepeat.java
index 1ef2ff77d5afca6..8194667fb6df880 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRepeat.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRepeat.java
@@ -18,6 +18,7 @@
 package org.apache.doris.nereids.trees.plans.physical;
 
 import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DataTrait;
 import org.apache.doris.nereids.properties.LogicalProperties;
 import org.apache.doris.nereids.properties.PhysicalProperties;
 import org.apache.doris.nereids.trees.expressions.Expression;
@@ -190,4 +191,24 @@ public PhysicalRepeat<CHILD_TYPE> resetLogicalProperties() {
         return new PhysicalRepeat<>(groupingSets, outputExpressions, groupExpression,
                 null, physicalProperties, statistics, child());
     }
+
+    @Override
+    public void computeUnique(DataTrait.Builder builder) {
+        // don't generate unique slot
+    }
+
+    @Override
+    public void computeUniform(DataTrait.Builder builder) {
+        builder.addUniformSlot(child(0).getLogicalProperties().getTrait());
+    }
+
+    @Override
+    public void computeEqualSet(DataTrait.Builder builder) {
+        builder.addEqualSet(child().getLogicalProperties().getTrait());
+    }
+
+    @Override
+    public void computeFd(DataTrait.Builder builder) {
+        builder.addFuncDepsDG(child().getLogicalProperties().getTrait());
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSink.java
index b6b8e5294fbbd61..42ed5761f80ffeb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSink.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSink.java
@@ -18,6 +18,7 @@
 package org.apache.doris.nereids.trees.plans.physical;
 
 import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DataTrait;
 import org.apache.doris.nereids.properties.LogicalProperties;
 import org.apache.doris.nereids.properties.PhysicalProperties;
 import org.apache.doris.nereids.trees.expressions.NamedExpression;
@@ -58,4 +59,24 @@ public List<Slot> computeOutput() {
     public List<NamedExpression> getOutputExprs() {
         return outputExprs;
     }
+
+    @Override
+    public void computeUnique(DataTrait.Builder builder) {
+        // should not be invoked
+    }
+
+    @Override
+    public void computeUniform(DataTrait.Builder builder) {
+        // should not be invoked
+    }
+
+    @Override
+    public void computeEqualSet(DataTrait.Builder builder) {
+        // should not be invoked
+    }
+
+    @Override
+    public void computeFd(DataTrait.Builder builder) {
+        // should not be invoked
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSqlCache.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSqlCache.java
index f0fde282011ddda..2b2c7ac04282214 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSqlCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSqlCache.java
@@ -28,6 +28,7 @@
 import org.apache.doris.nereids.properties.PhysicalProperties;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation;
 import org.apache.doris.nereids.trees.plans.ComputeResultSet;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.PlanType;
@@ -48,7 +49,8 @@
 import java.util.Optional;
 
 /** PhysicalSqlCache */
-public class PhysicalSqlCache extends PhysicalLeaf implements SqlCache, TreeStringPlan, ComputeResultSet {
+public class PhysicalSqlCache extends PhysicalLeaf
+        implements SqlCache, TreeStringPlan, ComputeResultSet, BlockFuncDepsPropagation {
     private final TUniqueId queryId;
     private final List<String> columnLabels;
     private final List<FieldInfo> fieldInfos;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java
index ce7d32f9cfe2928..fdb181137297852 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java
@@ -23,6 +23,7 @@
 import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.SlotReference;
 import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction;
+import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.PlanType;
 import org.apache.doris.nereids.trees.plans.RelationId;
@@ -38,7 +39,7 @@
 import java.util.Optional;
 
 /** PhysicalTableValuedFunctionRelation */
-public class PhysicalTVFRelation extends PhysicalRelation implements TVFRelation {
+public class PhysicalTVFRelation extends PhysicalRelation implements TVFRelation, BlockFuncDepsPropagation {
 
     private final TableValuedFunction function;
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTopN.java
index 014daa7bbcf7780..eeafbebd4581929 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTopN.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTopN.java
@@ -18,6 +18,7 @@
 package org.apache.doris.nereids.trees.plans.physical;
 
 import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DataTrait;
 import org.apache.doris.nereids.properties.LogicalProperties;
 import org.apache.doris.nereids.properties.OrderKey;
 import org.apache.doris.nereids.properties.PhysicalProperties;
@@ -165,6 +166,34 @@ public PhysicalTopN<Plan> resetLogicalProperties() {
                 null, physicalProperties, statistics, child());
     }
 
+    @Override
+    public void computeUnique(DataTrait.Builder builder) {
+        if (getLimit() == 1) {
+            getOutput().forEach(builder::addUniqueSlot);
+        } else {
+            builder.addUniqueSlot(child(0).getLogicalProperties().getTrait());
+        }
+    }
+
+    @Override
+    public void computeUniform(DataTrait.Builder builder) {
+        if (getLimit() == 1) {
+            getOutput().forEach(builder::addUniformSlot);
+        } else {
+            builder.addUniformSlot(child(0).getLogicalProperties().getTrait());
+        }
+    }
+
+    @Override
+    public void computeEqualSet(DataTrait.Builder builder) {
+        builder.addEqualSet(child(0).getLogicalProperties().getTrait());
+    }
+
+    @Override
+    public void computeFd(DataTrait.Builder builder) {
+        builder.addFuncDepsDG(child().getLogicalProperties().getTrait());
+    }
+
     @Override
     public ObjectId getObjectId() {
         return id;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalUnion.java
index 2a81698812a3c7b..36fea0056a0e813 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalUnion.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalUnion.java
@@ -18,9 +18,11 @@
 package org.apache.doris.nereids.trees.plans.physical;
 
 import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DataTrait;
 import org.apache.doris.nereids.properties.LogicalProperties;
 import org.apache.doris.nereids.properties.PhysicalProperties;
 import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.SlotReference;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.PlanType;
@@ -30,10 +32,16 @@
 import org.apache.doris.statistics.Statistics;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.Set;
 
 /**
  * Physical Union.
@@ -127,4 +135,92 @@ public PhysicalUnion resetLogicalProperties() {
         return new PhysicalUnion(qualifier, outputs, regularChildrenOutputs, constantExprsList,
                 Optional.empty(), null, physicalProperties, statistics, children);
     }
+
+    @Override
+    public void computeUnique(DataTrait.Builder builder) {
+        if (qualifier == Qualifier.DISTINCT) {
+            builder.addUniqueSlot(ImmutableSet.copyOf(getOutput()));
+        }
+    }
+
+    @Override
+    public void computeUniform(DataTrait.Builder builder) {
+        // don't propagate uniform slots
+    }
+
+    private List<BitSet> mapSlotToIndex(Plan plan, List<Set<Slot>> equalSlotsList) {
+        Map<Slot, Integer> slotToIndex = new HashMap<>();
+        for (int i = 0; i < plan.getOutput().size(); i++) {
+            slotToIndex.put(plan.getOutput().get(i), i);
+        }
+        List<BitSet> equalSlotIndicesList = new ArrayList<>();
+        for (Set<Slot> equalSlots : equalSlotsList) {
+            BitSet equalSlotIndices = new BitSet();
+            for (Slot slot : equalSlots) {
+                if (slotToIndex.containsKey(slot)) {
+                    equalSlotIndices.set(slotToIndex.get(slot));
+                }
+            }
+            if (equalSlotIndices.cardinality() > 1) {
+                equalSlotIndicesList.add(equalSlotIndices);
+            }
+        }
+        return equalSlotIndicesList;
+    }
+
+    @Override
+    public void computeEqualSet(DataTrait.Builder builder) {
+        if (children.isEmpty()) {
+            return;
+        }
+
+        // Get the list of equal slot sets and their corresponding index mappings for the first child
+        List<Set<Slot>> childEqualSlotsList = child(0).getLogicalProperties()
+                .getTrait().calAllEqualSet();
+        List<BitSet> childEqualSlotsIndicesList = mapSlotToIndex(child(0), childEqualSlotsList);
+        List<BitSet> unionEqualSlotIndicesList = new ArrayList<>(childEqualSlotsIndicesList);
+
+        // Traverse all children and find the equal sets that exist in all children
+        for (int i = 1; i < children.size(); i++) {
+            Plan child = children.get(i);
+
+            // Get the equal slot sets for the current child
+            childEqualSlotsList = child.getLogicalProperties().getTrait().calAllEqualSet();
+
+            // Map slots to indices for the current child
+            childEqualSlotsIndicesList = mapSlotToIndex(child, childEqualSlotsList);
+
+            // Only keep the equal pairs that exist in all children of the union
+            // This is done by calculating the intersection of all children's equal slot indices
+            for (BitSet unionEqualSlotIndices : unionEqualSlotIndicesList) {
+                BitSet intersect = new BitSet();
+                for (BitSet childEqualSlotIndices : childEqualSlotsIndicesList) {
+                    if (unionEqualSlotIndices.intersects(childEqualSlotIndices)) {
+                        intersect = childEqualSlotIndices;
+                        break;
+                    }
+                }
+                unionEqualSlotIndices.and(intersect);
+            }
+        }
+
+        // Build the functional dependencies for the output slots
+        List<Slot> outputList = getOutput();
+        for (BitSet equalSlotIndices : unionEqualSlotIndicesList) {
+            if (equalSlotIndices.cardinality() <= 1) {
+                continue;
+            }
+            int first = equalSlotIndices.nextSetBit(0);
+            int next = equalSlotIndices.nextSetBit(first + 1);
+            while (next > 0) {
+                builder.addEqualPair(outputList.get(first), outputList.get(next));
+                next = equalSlotIndices.nextSetBit(next + 1);
+            }
+        }
+    }
+
+    @Override
+    public void computeFd(DataTrait.Builder builder) {
+        // don't generate
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalWindow.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalWindow.java
index 7e6fd48f02da6d0..5d8188d4bc95b29 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalWindow.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalWindow.java
@@ -18,6 +18,7 @@
 package org.apache.doris.nereids.trees.plans.physical;
 
 import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DataTrait;
 import org.apache.doris.nereids.properties.LogicalProperties;
 import org.apache.doris.nereids.properties.PhysicalProperties;
 import org.apache.doris.nereids.properties.RequireProperties;
@@ -26,6 +27,10 @@
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.NamedExpression;
 import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.WindowExpression;
+import org.apache.doris.nereids.trees.expressions.functions.window.DenseRank;
+import org.apache.doris.nereids.trees.expressions.functions.window.Rank;
+import org.apache.doris.nereids.trees.expressions.functions.window.RowNumber;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.PlanType;
 import org.apache.doris.nereids.trees.plans.algebra.Window;
@@ -35,6 +40,7 @@
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 
 import java.util.List;
 import java.util.Objects;
@@ -189,4 +195,81 @@ public PhysicalWindow<CHILD_TYPE> resetLogicalProperties() {
         return new PhysicalWindow<>(windowFrameGroup, requireProperties, windowExpressions, groupExpression,
                 null, physicalProperties, statistics, child());
     }
+
+    private boolean isUnique(NamedExpression namedExpression) {
+        if (namedExpression.children().size() != 1 || !(namedExpression.child(0) instanceof WindowExpression)) {
+            return false;
+        }
+        WindowExpression windowExpr = (WindowExpression) namedExpression.child(0);
+        List<Expression> partitionKeys = windowExpr.getPartitionKeys();
+        // Now we only support slot type keys
+        if (!partitionKeys.stream().allMatch(Slot.class::isInstance)) {
+            return false;
+        }
+        ImmutableSet<Slot> slotSet = partitionKeys.stream()
+                .map(s -> (Slot) s)
+                .collect(ImmutableSet.toImmutableSet());
+        // if partition by keys are uniform or empty, output is unique
+        if (child(0).getLogicalProperties().getTrait().isUniformAndNotNull(slotSet)
+                || slotSet.isEmpty()) {
+            if (windowExpr.getFunction() instanceof RowNumber) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private boolean isUniform(NamedExpression namedExpression) {
+        if (namedExpression.children().size() != 1 || !(namedExpression.child(0) instanceof WindowExpression)) {
+            return false;
+        }
+        WindowExpression windowExpr = (WindowExpression) namedExpression.child(0);
+        List<Expression> partitionKeys = windowExpr.getPartitionKeys();
+        // Now we only support slot type keys
+        if (!partitionKeys.stream().allMatch(Slot.class::isInstance)) {
+            return false;
+        }
+        ImmutableSet<Slot> slotSet = partitionKeys.stream()
+                .map(s -> (Slot) s)
+                .collect(ImmutableSet.toImmutableSet());
+        // if partition by keys are unique, output is uniform
+        if (child(0).getLogicalProperties().getTrait().isUniqueAndNotNull(slotSet)) {
+            if (windowExpr.getFunction() instanceof RowNumber
+                    || windowExpr.getFunction() instanceof Rank
+                    || windowExpr.getFunction() instanceof DenseRank) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    @Override
+    public void computeUnique(DataTrait.Builder builder) {
+        builder.addUniqueSlot(child(0).getLogicalProperties().getTrait());
+        for (NamedExpression namedExpression : windowExpressions) {
+            if (isUnique(namedExpression)) {
+                builder.addUniqueSlot(namedExpression.toSlot());
+            }
+        }
+    }
+
+    @Override
+    public void computeUniform(DataTrait.Builder builder) {
+        builder.addUniformSlot(child(0).getLogicalProperties().getTrait());
+        for (NamedExpression namedExpression : windowExpressions) {
+            if (isUniform(namedExpression)) {
+                builder.addUniformSlot(namedExpression.toSlot());
+            }
+        }
+    }
+
+    @Override
+    public void computeEqualSet(DataTrait.Builder builder) {
+        builder.addEqualSet(child(0).getLogicalProperties().getTrait());
+    }
+
+    @Override
+    public void computeFd(DataTrait.Builder builder) {
+        builder.addFuncDepsDG(child().getLogicalProperties().getTrait());
+    }
 }
diff --git a/regression-test/data/nereids_p0/runtime_filter/runtime_filter.out b/regression-test/data/nereids_p0/runtime_filter/runtime_filter.out
new file mode 100644
index 000000000000000..6cf5d5a5b4c6e2f
--- /dev/null
+++ b/regression-test/data/nereids_p0/runtime_filter/runtime_filter.out
@@ -0,0 +1,19 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !check_no_rf --
+PhysicalResultSink
+--PhysicalDistribute[DistributionSpecGather]
+----hashJoin[INNER_JOIN broadcast] hashCondition=((lineitem.l_orderkey = orders.o_orderkey)) otherCondition=()
+------filter((lineitem.l_orderkey = 1))
+--------PhysicalOlapScan[lineitem]
+------filter((orders.o_orderkey = 1))
+--------PhysicalOlapScan[orders]
+
+-- !check_one_rf --
+PhysicalResultSink
+--PhysicalDistribute[DistributionSpecGather]
+----hashJoin[INNER_JOIN broadcast] hashCondition=((lineitem.l_linenumber = orders.o_v) and (lineitem.l_orderkey = orders.o_orderkey)) otherCondition=() build RFs:RF0 o_v->[l_linenumber]
+------filter((lineitem.l_orderkey = 1))
+--------PhysicalOlapScan[lineitem] apply RFs: RF0
+------filter((orders.o_orderkey = 1))
+--------PhysicalOlapScan[orders]
+
diff --git a/regression-test/data/nereids_rules_p0/cte/test_cte_filter_pushdown.out b/regression-test/data/nereids_rules_p0/cte/test_cte_filter_pushdown.out
index 7dd6492aa12499c..0bbae0dc25f3a1d 100644
--- a/regression-test/data/nereids_rules_p0/cte/test_cte_filter_pushdown.out
+++ b/regression-test/data/nereids_rules_p0/cte/test_cte_filter_pushdown.out
@@ -7,9 +7,9 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --------filter((main.k1 = 1))
 ----------PhysicalOlapScan[test]
 --PhysicalResultSink
-----hashJoin[INNER_JOIN] hashCondition=((m1.k1 = m2.k1)) otherCondition=() build RFs:RF0 k1->[k1]
+----hashJoin[INNER_JOIN] hashCondition=((m1.k1 = m2.k1)) otherCondition=()
 ------filter((temp.k1 = 1))
---------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF0
+--------PhysicalCteConsumer ( cteId=CTEId#0 )
 ------filter((m2.k1 = 1))
 --------PhysicalCteConsumer ( cteId=CTEId#0 )
 
@@ -21,9 +21,9 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --------PhysicalQuickSort[LOCAL_SORT]
 ----------PhysicalOlapScan[test]
 --PhysicalResultSink
-----hashJoin[INNER_JOIN] hashCondition=((m1.k1 = m2.k1)) otherCondition=() build RFs:RF0 k1->[k1]
+----hashJoin[INNER_JOIN] hashCondition=((m1.k1 = m2.k1)) otherCondition=()
 ------filter((temp.k1 = 1))
---------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF0
+--------PhysicalCteConsumer ( cteId=CTEId#0 )
 ------filter((m2.k1 = 1))
 --------PhysicalCteConsumer ( cteId=CTEId#0 )
 
diff --git a/regression-test/data/nereids_tpch_p0/tpch/runtime_filter.out b/regression-test/data/nereids_tpch_p0/tpch/runtime_filter.out
new file mode 100644
index 000000000000000..54686c33c864398
--- /dev/null
+++ b/regression-test/data/nereids_tpch_p0/tpch/runtime_filter.out
@@ -0,0 +1,29 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !no_rf --
+PhysicalResultSink
+--hashAgg[GLOBAL]
+----PhysicalDistribute[DistributionSpecGather]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------hashJoin[INNER_JOIN broadcast] hashCondition=((lineitem.l_orderkey = orders.o_orderkey)) otherCondition=()
+------------PhysicalProject
+--------------filter((lineitem.l_orderkey = 1))
+----------------PhysicalOlapScan[lineitem]
+------------PhysicalProject
+--------------filter((orders.o_orderkey = 1))
+----------------PhysicalOlapScan[orders]
+
+-- !one_rf --
+PhysicalResultSink
+--hashAgg[GLOBAL]
+----PhysicalDistribute[DistributionSpecGather]
+------hashAgg[LOCAL]
+--------PhysicalProject
+----------hashJoin[INNER_JOIN broadcast] hashCondition=((lineitem.l_orderkey = orders.o_orderkey) and (lineitem.l_suppkey = orders.o_custkey)) otherCondition=() build RFs:RF0 o_custkey->[l_suppkey];RF1 o_custkey->[l_suppkey];RF2 o_custkey->[l_suppkey]
+------------PhysicalProject
+--------------filter((lineitem.l_orderkey = 1))
+----------------PhysicalOlapScan[lineitem] apply RFs: RF0 RF1 RF2
+------------PhysicalProject
+--------------filter((orders.o_orderkey = 1))
+----------------PhysicalOlapScan[orders]
+
diff --git a/regression-test/suites/nereids_p0/runtime_filter/runtime_filter.groovy b/regression-test/suites/nereids_p0/runtime_filter/runtime_filter.groovy
index 17fac79427e839c..3331b0250e5d509 100644
--- a/regression-test/suites/nereids_p0/runtime_filter/runtime_filter.groovy
+++ b/regression-test/suites/nereids_p0/runtime_filter/runtime_filter.groovy
@@ -16,6 +16,7 @@
 // under the License.
 
 suite("runtime_filter") {
+    sql " set enable_parallel_result_sink=false" 
     sql ''' drop table if exists rf_dws_asset_domain_statistics_daily'''
     sql '''CREATE TABLE rf_dws_asset_domain_statistics_daily (
         account_id int(11) NULL,
@@ -45,4 +46,41 @@ suite("runtime_filter") {
             """
         notContains("RFs")
     }
+
+    sql """
+      drop table if exists lineitem;
+      CREATE TABLE IF NOT EXISTS lineitem (
+        L_ORDERKEY    INTEGER NOT NULL,
+        L_LINENUMBER  INTEGER NOT NULL
+        )
+        DUPLICATE KEY(L_ORDERKEY)
+        DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 3
+        PROPERTIES (
+        "replication_num" = "1"
+        );
+      insert into lineitem values (1, 1), (1, 2), (1, 3), (2,1);
+      
+      drop table if exists orders;
+      CREATE TABLE IF NOT EXISTS orders (
+        O_ORDERKEY    INTEGER NOT NULL,
+        O_V  INTEGER NOT NULL
+        )
+        DUPLICATE KEY(O_ORDERKEY)
+        DISTRIBUTED BY HASH(O_ORDERKEY) BUCKETS 3
+        PROPERTIES (
+        "replication_num" = "1"
+        );
+      insert into orders values(1, 2);
+
+      set disable_join_reorder=true;
+      set runtime_filter_type=2;
+    """
+
+    qt_check_no_rf """
+      explain shape plan select * from lineitem join orders on l_orderkey=o_orderkey where o_orderkey=1;
+    """
+
+    qt_check_one_rf """
+      explain shape plan select * from lineitem join orders on l_orderkey=o_orderkey and l_linenumber=o_v where o_orderkey=1;
+    """
 }
\ No newline at end of file

From d4648afb5445360bdcb6148255eac41cd267429a Mon Sep 17 00:00:00 2001
From: Thearas <gaozifeng@selectdb.com>
Date: Wed, 8 Jan 2025 22:18:37 +0800
Subject: [PATCH 052/140] [opt](hive docker)Parallel put hive data (#46571)

### What problem does this PR solve?

Problem Summary:
Parallel put `tpch1.db`, `paimon1` and `tvf_data` hive data. Reduce the
time cost from 22m to 16m on 16C machine.
---
 .../hive/scripts/hive-metastore.sh            | 49 ++++++++++++-------
 .../thirdparties/run-thirdparties-docker.sh   |  2 +-
 2 files changed, 33 insertions(+), 18 deletions(-)

diff --git a/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh b/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh
index 7ff6bc4c62f2fe6..6681a5130664ef3 100755
--- a/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh
+++ b/docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh
@@ -18,6 +18,8 @@
 
 set -e -x
 
+parallel=$(getconf _NPROCESSORS_ONLN)
+
 nohup /opt/hive/bin/hive --service metastore &
 
 # wait metastore start
@@ -37,7 +39,7 @@ done
 touch "${lockfile1}"
 
 DATA_DIR="/mnt/scripts/data/"
-find "${DATA_DIR}" -type f -name "run.sh" -print0 | xargs -0 -n 1 -P 10 -I {} sh -c '
+find "${DATA_DIR}" -type f -name "run.sh" -print0 | xargs -0 -n 1 -P "${parallel}" -I {} sh -c '
     START_TIME=$(date +%s)
     chmod +x "{}" && "{}"
     END_TIME=$(date +%s)
@@ -92,45 +94,58 @@ fi
 rm -f "${lockfile2}"
 
 # put data file
+hadoop_put_pids=()
+hadoop fs -mkdir -p /user/doris/
+
+
 ## put tpch1
 if [[ -z "$(ls /mnt/scripts/tpch1.db)" ]]; then
     echo "tpch1.db does not exist"
     exit 1
 fi
-hadoop fs -mkdir -p /user/doris/
-hadoop fs -put /mnt/scripts/tpch1.db /user/doris/
-if [[ -z "$(hadoop fs -ls /user/doris/tpch1.db)" ]]; then
-    echo "tpch1.db put failed"
-    exit 1
-fi
+hadoop fs -copyFromLocal -f /mnt/scripts/tpch1.db /user/doris/ &
+hadoop_put_pids+=($!)
 
 ## put paimon1
 if [[ -z "$(ls /mnt/scripts/paimon1)" ]]; then
     echo "paimon1 does not exist"
     exit 1
 fi
-hadoop fs -put /mnt/scripts/paimon1 /user/doris/
-if [[ -z "$(hadoop fs -ls /user/doris/paimon1)" ]]; then
-    echo "paimon1 put failed"
-    exit 1
-fi
+hadoop fs -copyFromLocal -f /mnt/scripts/paimon1 /user/doris/ &
+hadoop_put_pids+=($!)
 
 ## put tvf_data
 if [[ -z "$(ls /mnt/scripts/tvf_data)" ]]; then
     echo "tvf_data does not exist"
     exit 1
 fi
-hadoop fs -put /mnt/scripts/tvf_data /user/doris/
+hadoop fs -copyFromLocal -f /mnt/scripts/tvf_data /user/doris/ &
+hadoop_put_pids+=($!)
+
+## put other preinstalled data
+hadoop fs -copyFromLocal -f /mnt/scripts/preinstalled_data /user/doris/ &
+hadoop_put_pids+=($!)
+
+
+# wait put finish
+set +e
+wait "${hadoop_put_pids[@]}"
+set -e
+if [[ -z "$(hadoop fs -ls /user/doris/paimon1)" ]]; then
+    echo "paimon1 put failed"
+    exit 1
+fi
+if [[ -z "$(hadoop fs -ls /user/doris/tpch1.db)" ]]; then
+    echo "tpch1.db put failed"
+    exit 1
+fi
 if [[ -z "$(hadoop fs -ls /user/doris/tvf_data)" ]]; then
     echo "tvf_data put failed"
     exit 1
 fi
 
-## put other preinstalled data
-hadoop fs -put /mnt/scripts/preinstalled_data /user/doris/
-
 # create tables
-ls /mnt/scripts/create_preinstalled_scripts/*.hql | xargs -n 1 -P 10 -I {} bash -c '
+ls /mnt/scripts/create_preinstalled_scripts/*.hql | xargs -n 1 -P "${parallel}" -I {} bash -c '
     START_TIME=$(date +%s)
     hive -f {}
     END_TIME=$(date +%s)
diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh
index d00fdcea3aa20c9..16c8b9e7d6b5505 100755
--- a/docker/thirdparties/run-thirdparties-docker.sh
+++ b/docker/thirdparties/run-thirdparties-docker.sh
@@ -614,7 +614,7 @@ start_minio() {
     fi
 }
 
-echo "starting dockers in parrallel"
+echo "starting dockers in parallel"
 
 declare -A pids
 

From f7560a6ce723e36a1fd5c6528d0f33289f37a4d0 Mon Sep 17 00:00:00 2001
From: walter <maochuan@selectdb.com>
Date: Wed, 8 Jan 2025 22:34:02 +0800
Subject: [PATCH 053/140] [fix](suites) Fix atomic restore alter suite with
 master_sql (#46550)

Add a helper `master_sql`, to connect to FE master and execute sql.
---
 .../doris/regression/suite/Suite.groovy       | 12 +++++
 .../regression/suite/SuiteContext.groovy      | 47 ++++++++++++++++++-
 ...st_backup_restore_atomic_with_alter.groovy |  4 +-
 3 files changed, 60 insertions(+), 3 deletions(-)

diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index 7f21f45774bd6ea..3995371db952da6 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -403,6 +403,10 @@ class Suite implements GroovyInterceptable {
         }
     }
 
+    List<List<Object>> master_sql(String sqlStr, boolean isOrder = false) {
+        return sql_impl(context.getMasterConnection(), sqlStr, isOrder)
+    }
+
     List<List<Object>> multi_sql(String sqlStr, boolean isOrder = false) {
         String[] sqls = sqlStr.split(";")
         def result = new ArrayList<Object>();
@@ -514,6 +518,10 @@ class Suite implements GroovyInterceptable {
         return sql_return_maparray_impl(sqlStr, context.getConnection())
     }
 
+    def master_sql_return_maparray(String sqlStr) {
+        return sql_return_maparray_impl(sqlStr, context.getMasterConnection())
+    }
+
     def arrow_flight_sql_return_maparray(String sqlStr) {
         return sql_return_maparray_impl((String) ("USE ${context.dbName};" + sqlStr), context.getArrowFlightSqlConnection())
     }
@@ -1268,6 +1276,8 @@ class Suite implements GroovyInterceptable {
                     tupleResult = JdbcUtils.executeToStringList(context.getArrowFlightSqlConnection(), (PreparedStatement) arg)
                 } else if (tag.contains("target_sql")) {
                     tupleResult = JdbcUtils.executeToStringList(context.getTargetConnection(this), (PreparedStatement) arg)
+                } else if (tag.contains("master_sql")) {
+                    tupleResult = JdbcUtils.executeToStringList(context.getMasterConnection(), (PreparedStatement) arg)
                 } else {
                     tupleResult = JdbcUtils.executeToStringList(context.getConnection(),  (PreparedStatement) arg)
                 }
@@ -1281,6 +1291,8 @@ class Suite implements GroovyInterceptable {
                             (String) ("USE ${context.dbName};" + (String) arg))
                 } else if (tag.contains("target_sql")) {
                     tupleResult = JdbcUtils.executeToStringList(context.getTargetConnection(this), (String) arg)
+                } else if (tag.contains("master_sql")) {
+                    tupleResult = JdbcUtils.executeToStringList(context.getMasterConnection(), (PreparedStatement) arg)
                 } else {
                     tupleResult = JdbcUtils.executeToStringList(context.getConnection(),  (String) arg)
                 }
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy
index 88ad1fac355d176..076566651e5429b 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy
@@ -44,6 +44,7 @@ class SuiteContext implements Closeable {
     public final String group
     public final String dbName
     public final ThreadLocal<ConnectionInfo> threadLocalConn = new ThreadLocal<>()
+    public final ThreadLocal<ConnectionInfo> threadLocalMasterConn = new ThreadLocal<>()
     public final ThreadLocal<ConnectionInfo> threadArrowFlightSqlConn = new ThreadLocal<>()
     public final ThreadLocal<Connection> threadHive2DockerConn = new ThreadLocal<>()
     public final ThreadLocal<Connection> threadHive3DockerConn = new ThreadLocal<>()
@@ -145,13 +146,26 @@ class SuiteContext implements Closeable {
         if (threadConnInfo == null) {
             threadConnInfo = new ConnectionInfo()
             threadConnInfo.conn = config.getConnectionByDbName(dbName)
-            threadConnInfo.username = config.jdbcUser 
+            threadConnInfo.username = config.jdbcUser
             threadConnInfo.password = config.jdbcPassword
             threadLocalConn.set(threadConnInfo)
         }
         return threadConnInfo.conn
     }
 
+    // like getConnection, but connect to FE master
+    Connection getMasterConnection() {
+        def threadConnInfo = threadLocalMasterConn.get()
+        if (threadConnInfo == null) {
+            threadConnInfo = new ConnectionInfo()
+            threadConnInfo.conn = getMasterConnectionByDbName(dbName)
+            threadConnInfo.username = config.jdbcUser
+            threadConnInfo.password = config.jdbcPassword
+            threadLocalMasterConn.set(threadConnInfo)
+        }
+        return threadConnInfo.conn
+    }
+
     Connection getArrowFlightSqlConnection() {
         def threadConnInfo = threadArrowFlightSqlConn.get()
         if (threadConnInfo == null) {
@@ -316,6 +330,27 @@ class SuiteContext implements Closeable {
         }
     }
 
+    Connection getMasterConnectionByDbName(String dbName) {
+        def result = JdbcUtils.executeToMapArray(getConnection(), "SHOW FRONTENDS")
+        def master = null
+        for (def row : result) {
+            if (row.IsMaster == "true") {
+                master = row
+                break
+            }
+        }
+        if (master) {
+            log.info("master found: ${master.Host}:${master.HttpPort}")
+            def url = Config.buildUrlWithDb(master.Host as String, master.QueryPort as Integer, dbName)
+            def username = config.jdbcUser
+            def password = config.jdbcPassword
+
+            return DriverManager.getConnection(url, username, password)
+        } else {
+            throw new Exception("No master found to reconnect")
+        }
+    }
+
     def reconnectToMasterFe = { ->
         log.info("Reconnecting to a new master frontend...")
         def result = JdbcUtils.executeToMapArray(getConnection(), "SHOW FRONTENDS")
@@ -468,6 +503,16 @@ class SuiteContext implements Closeable {
             }
         }
 
+        ConnectionInfo master_conn = threadLocalMasterConn.get()
+        if (master_conn != null) {
+            threadLocalMasterConn.remove()
+            try {
+                master_conn.conn.close()
+            } catch (Throwable t) {
+                log.warn("Close master connection failed", t)
+            }
+        }
+
         ConnectionInfo arrow_flight_sql_conn = threadArrowFlightSqlConn.get()
         if (arrow_flight_sql_conn != null) {
             threadArrowFlightSqlConn.remove()
diff --git a/regression-test/suites/backup_restore/test_backup_restore_atomic_with_alter.groovy b/regression-test/suites/backup_restore/test_backup_restore_atomic_with_alter.groovy
index ea8ddd4d11c1d94..6de54eafca5db7e 100644
--- a/regression-test/suites/backup_restore/test_backup_restore_atomic_with_alter.groovy
+++ b/regression-test/suites/backup_restore/test_backup_restore_atomic_with_alter.groovy
@@ -125,7 +125,7 @@ suite("test_backup_restore_atomic_with_alter", "backup_restore") {
     sql "SYNC"
 
     // 0. table_1 has in_atomic_restore property
-    def show_result = sql """ SHOW CREATE TABLE ${dbName}.${tableNamePrefix}_1 """
+    def show_result = master_sql """ SHOW CREATE TABLE ${dbName}.${tableNamePrefix}_1 """
     logger.info("SHOW CREATE TABLE ${tableNamePrefix}_1: ${show_result}")
     assertTrue(show_result[0][1].contains("in_atomic_restore"))
 
@@ -230,7 +230,7 @@ suite("test_backup_restore_atomic_with_alter", "backup_restore") {
     sql "SYNC"
 
     // 5. The restore job is cancelled, the in_atomic_restore property has been removed.
-    show_result = sql """ SHOW CREATE TABLE ${dbName}.${tableNamePrefix}_1 """
+    show_result = master_sql """ SHOW CREATE TABLE ${dbName}.${tableNamePrefix}_1 """
     logger.info("SHOW CREATE TABLE ${tableNamePrefix}_1: ${show_result}")
     assertFalse(show_result[0][1].contains("in_atomic_restore"))
 

From c5953f8d41de69512ffe986d39161feb89b9d1aa Mon Sep 17 00:00:00 2001
From: Lei Zhang <zhanglei@selectdb.com>
Date: Wed, 8 Jan 2025 22:45:23 +0800
Subject: [PATCH 054/140] [fix](vault) fix `CreateTableLikeStmt` cannot work in
 stoarge vault mode (#46625)

```
  Failed to execute CREATE TABLE LIKE demo_table. Reason: errCode = 2, detailMessage = Unknown properties
```
---
 .../doris/common/util/PropertyAnalyzer.java   | 45 +++++++++++++++++--
 .../doris/datasource/InternalCatalog.java     | 42 ++++-------------
 .../test_vault_privilege_with_role.groovy     | 20 ++++-----
 3 files changed, 60 insertions(+), 47 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index 536ca5633bf30a0..a66a2ae44329476 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -34,6 +34,7 @@
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.Config;
 import org.apache.doris.common.DdlException;
+import org.apache.doris.common.Pair;
 import org.apache.doris.datasource.CatalogIf;
 import org.apache.doris.datasource.ExternalCatalog;
 import org.apache.doris.policy.Policy;
@@ -1183,14 +1184,50 @@ public static String analyzeStoragePolicy(Map<String, String> properties) {
         return storagePolicy;
     }
 
-    public static String analyzeStorageVault(Map<String, String> properties) {
-        String storageVault = null;
+    /**
+     * @param properties
+     * @return <storageVaultName, storageVaultId>
+     * @throws AnalysisException
+     */
+    public static Pair<String, String> analyzeStorageVault(Map<String, String> properties) throws AnalysisException {
+        String storageVaultName = null;
         if (properties != null && properties.containsKey(PROPERTIES_STORAGE_VAULT_NAME)) {
-            storageVault = properties.get(PROPERTIES_STORAGE_VAULT_NAME);
+            storageVaultName = properties.get(PROPERTIES_STORAGE_VAULT_NAME);
             properties.remove(PROPERTIES_STORAGE_VAULT_NAME);
         }
 
-        return storageVault;
+        if (Strings.isNullOrEmpty(storageVaultName)) {
+            // If user does not specify one storage vault then FE would use the default vault
+            Pair<String, String> info = Env.getCurrentEnv().getStorageVaultMgr().getDefaultStorageVault();
+            if (info == null) {
+                throw new AnalysisException("No default storage vault."
+                        + " You can use `SHOW STORAGE VAULT` to get all available vaults,"
+                        + " and pick one set default vault with `SET <vault_name> AS DEFAULT STORAGE VAULT`");
+            }
+            storageVaultName = info.first;
+            LOG.info("Using default storage vault, name:{} id:{}", info.first, info.second);
+        }
+
+        if (Strings.isNullOrEmpty(storageVaultName)) {
+            throw new AnalysisException("Invalid Storage Vault. "
+                    + " You can use `SHOW STORAGE VAULT` to get all available vaults,"
+                    + " and pick one to set the table property `\"storage_vault_name\" = \"<vault_name>\"`");
+        }
+
+        String storageVaultId = Env.getCurrentEnv().getStorageVaultMgr().getVaultIdByName(storageVaultName);
+        if (Strings.isNullOrEmpty(storageVaultId)) {
+            throw new AnalysisException("Storage vault '" + storageVaultName + "' does not exist. "
+                    + "You can use `SHOW STORAGE VAULT` to get all available vaults, "
+                    + "or create a new one with `CREATE STORAGE VAULT`.");
+        }
+
+        if (properties != null && properties.containsKey(PROPERTIES_STORAGE_VAULT_ID)) {
+            Preconditions.checkArgument(storageVaultId.equals(properties.get(PROPERTIES_STORAGE_VAULT_ID)),
+                    "storageVaultId check failed, %s-%s", storageVaultId, properties.get(PROPERTIES_STORAGE_VAULT_ID));
+            properties.remove(PROPERTIES_STORAGE_VAULT_ID);
+        }
+
+        return Pair.of(storageVaultName, storageVaultId);
     }
 
     // analyze property like : "type" = "xxx";
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index bfffd3e154b3b1e..5b3c106bbc8e525 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -2736,44 +2736,20 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx
         olapTable.setEnableSingleReplicaCompaction(enableSingleReplicaCompaction);
 
         if (Config.isCloudMode() && ((CloudEnv) env).getEnableStorageVault()) {
-            // set storage vault
-            String storageVaultName = PropertyAnalyzer.analyzeStorageVault(properties);
-            String storageVaultId = null;
-            // If user does not specify one storage vault then FE would use the default vault
-            if (Strings.isNullOrEmpty(storageVaultName)) {
-                Pair<String, String> info = env.getStorageVaultMgr().getDefaultStorageVault();
-                if (info != null) {
-                    storageVaultName = info.first;
-                    storageVaultId = info.second;
-                    LOG.info("Using default storage vault: name={}, id={}", storageVaultName, storageVaultId);
-                } else {
-                    throw new DdlException("No default storage vault."
-                            + " You can use `SHOW STORAGE VAULT` to get all available vaults,"
-                            + " and pick one set default vault with `SET <vault_name> AS DEFAULT STORAGE VAULT`");
-                }
-            }
-
-            if (storageVaultName == null || storageVaultName.isEmpty()) {
-                throw new DdlException("Invalid Storage Vault. "
-                        + " You can use `SHOW STORAGE VAULT` to get all available vaults,"
-                        + " and pick one to set the table property `\"storage_vault_name\" = \"<vault_name>\"`");
-            }
+            // <storageVaultName, storageVaultId>
+            Pair<String, String> storageVaultInfoPair = PropertyAnalyzer.analyzeStorageVault(properties);
 
             // Check if user has storage vault usage privilege
-            if (ctx != null && !env.getAuth()
-                    .checkStorageVaultPriv(ctx.getCurrentUserIdentity(), storageVaultName, PrivPredicate.USAGE)) {
+            if (ConnectContext.get() != null && !env.getAuth()
+                    .checkStorageVaultPriv(ctx.getCurrentUserIdentity(),
+                            storageVaultInfoPair.first, PrivPredicate.USAGE)) {
                 throw new DdlException("USAGE denied to user '" + ConnectContext.get().getQualifiedUser()
                         + "'@'" + ConnectContext.get().getRemoteIP()
-                        + "' for storage vault '" + storageVaultName + "'");
-            }
-
-            storageVaultId = env.getStorageVaultMgr().getVaultIdByName(storageVaultName);
-            if (Strings.isNullOrEmpty(storageVaultId)) {
-                throw new DdlException("Storage vault '" + storageVaultName + "' does not exist. "
-                        + "You can use `SHOW STORAGE VAULT` to get all available vaults, "
-                        + "or create a new one with `CREATE STORAGE VAULT`.");
+                        + "' for storage vault '" + storageVaultInfoPair.first + "'");
             }
-            olapTable.setStorageVaultId(storageVaultId);
+            Preconditions.checkArgument(StringUtils.isNumeric(storageVaultInfoPair.second),
+                    "Invaild storage vault id :%s", storageVaultInfoPair.second);
+            olapTable.setStorageVaultId(storageVaultInfoPair.second);
         }
 
         // check `update on current_timestamp`
diff --git a/regression-test/suites/vault_p0/privilege/test_vault_privilege_with_role.groovy b/regression-test/suites/vault_p0/privilege/test_vault_privilege_with_role.groovy
index 54cace642d4188b..7192dc40aed816b 100644
--- a/regression-test/suites/vault_p0/privilege/test_vault_privilege_with_role.groovy
+++ b/regression-test/suites/vault_p0/privilege/test_vault_privilege_with_role.groovy
@@ -46,6 +46,16 @@ suite("test_vault_privilege_with_role", "nonConcurrent") {
     sql """CREATE USER ${userName} identified by '${userPassword}' DEFAULT ROLE '${roleName}'"""
     sql """GRANT create_priv ON *.*.* TO '${userName}'; """
 
+    sql """
+        CREATE STORAGE VAULT ${hdfsVaultName}
+        PROPERTIES (
+            "type"="HDFS",
+            "fs.defaultFS"="${getHmsHdfsFs()}",
+            "path_prefix" = "${hdfsVaultName}",
+            "hadoop.username" = "${getHmsUser()}"
+        );
+        """
+
     connect(userName, userPassword, context.config.jdbcUrl) {
         expectExceptionLike({
             sql """
@@ -65,16 +75,6 @@ suite("test_vault_privilege_with_role", "nonConcurrent") {
 
     sql """ GRANT usage_priv ON STORAGE VAULT '${hdfsVaultName}' TO ROLE '${roleName}';"""
 
-    sql """
-        CREATE STORAGE VAULT ${hdfsVaultName}
-        PROPERTIES (
-            "type"="HDFS",
-            "fs.defaultFS"="${getHmsHdfsFs()}",
-            "path_prefix" = "${hdfsVaultName}",
-            "hadoop.username" = "${getHmsUser()}"
-        );
-        """
-
     connect(userName, userPassword, context.config.jdbcUrl) {
         sql """
             CREATE TABLE IF NOT EXISTS ${dbName}.${tableName} (

From 12e94294dc75efe00f41f7ba01d2b487713525bf Mon Sep 17 00:00:00 2001
From: bobhan1 <baohan@selectdb.com>
Date: Wed, 8 Jan 2025 23:20:25 +0800
Subject: [PATCH 055/140] [Opt](bvar) Add bvar counter for
 `CALCULATE_DELETE_BITMAP` task (#46195)

Add bvar counter for CALCULATE_DELETE_BITMAP task
---
 be/src/agent/task_worker_pool.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp
index a8ab93de455c3b4..40b84facecfc5d1 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -454,6 +454,7 @@ bvar::Adder<uint64_t> CLONE_count("task", "CLONE");
 bvar::Adder<uint64_t> STORAGE_MEDIUM_MIGRATE_count("task", "STORAGE_MEDIUM_MIGRATE");
 bvar::Adder<uint64_t> GC_BINLOG_count("task", "GC_BINLOG");
 bvar::Adder<uint64_t> UPDATE_VISIBLE_VERSION_count("task", "UPDATE_VISIBLE_VERSION");
+bvar::Adder<uint64_t> CALCULATE_DELETE_BITMAP_count("task", "CALCULATE_DELETE_BITMAP");
 
 void add_task_count(const TAgentTaskRequest& task, int n) {
     // clang-format off
@@ -481,6 +482,7 @@ void add_task_count(const TAgentTaskRequest& task, int n) {
     ADD_TASK_COUNT(STORAGE_MEDIUM_MIGRATE)
     ADD_TASK_COUNT(GC_BINLOG)
     ADD_TASK_COUNT(UPDATE_VISIBLE_VERSION)
+    ADD_TASK_COUNT(CALCULATE_DELETE_BITMAP)
     #undef ADD_TASK_COUNT
     case TTaskType::REALTIME_PUSH:
     case TTaskType::PUSH:

From 0d784c05586cc47ec698ad3b522fd4ecb196619f Mon Sep 17 00:00:00 2001
From: minghong <zhouminghong@selectdb.com>
Date: Thu, 9 Jan 2025 10:14:31 +0800
Subject: [PATCH 056/140] [fix](regression) unstable case: partition_key_minmax
 (#46597)

### What problem does this PR solve?
root cause:
by default partition column stats analyze is disabled.
but in case partition_col_stats, it is turned on.
1. make partition_key_minmax support partition column stats
2. set nonConcurrent  for partition_col_stats
---
 .../nereids_p0/stats/partition_col_stats.groovy   |  2 +-
 .../nereids_p0/stats/partition_key_minmax.groovy  | 15 ++++++++++-----
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/regression-test/suites/nereids_p0/stats/partition_col_stats.groovy b/regression-test/suites/nereids_p0/stats/partition_col_stats.groovy
index a5ef7d4878e5db7..4508bcf415a49d3 100644
--- a/regression-test/suites/nereids_p0/stats/partition_col_stats.groovy
+++ b/regression-test/suites/nereids_p0/stats/partition_col_stats.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("partition_col_stats") {
+suite("partition_col_stats", "p0, nonConcurrent") {
     try {
         multi_sql """
             set global enable_partition_analyze=true;
diff --git a/regression-test/suites/nereids_p0/stats/partition_key_minmax.groovy b/regression-test/suites/nereids_p0/stats/partition_key_minmax.groovy
index e7d733611e79d1e..a7ca288ecf3d16d 100644
--- a/regression-test/suites/nereids_p0/stats/partition_key_minmax.groovy
+++ b/regression-test/suites/nereids_p0/stats/partition_key_minmax.groovy
@@ -28,7 +28,7 @@ suite("partition_key_minmax") {
         )
         distributed by hash(a) properties("replication_num"="1");
 
-        insert into rangetable values (5, 3, 0), (22, 150, 1), (333, 1, 2),(6, 1, 3);
+        insert into rangetable values (5, 3, 0),(6, 1, 3),(22, 150, 1),(333, 1, 2);
 
         analyze table rangetable with sync;
     """
@@ -38,8 +38,14 @@ suite("partition_key_minmax") {
         sql """memo plan
             select * from rangetable where a < 250;
             """
-        containsAny("a#0 -> ndv=2.6667, min=5.000000(5), max=30.000000(30), count=2.6667")
-        containsAny("a#0 -> ndv=3, min=5.000000(5), max=30.000000(30), count=2.6667")
+        // case 1: partition col stats not available, partition row count not available
+        containsAny("a#0 -> ndv=2.6667, min=5.000000(5), max=30.000000(30)")
+        // case 2: partition col stats not available, partition row count available
+        containsAny("a#0 -> ndv=3.0000, min=5.000000(5), max=30.000000(30)")
+        // case 3: partition col stats available, partition row count not available
+        containsAny("a#0 -> ndv=2.6667, min=5.000000(5), max=22.000000(22)")
+        // case 4: partition col stats available, partition row count available
+        containsAny("a#0 -> ndv=3.0000, min=5.000000(5), max=22.000000(22)")
     }
 
     sql """
@@ -65,8 +71,7 @@ suite("partition_key_minmax") {
         sql """
          memo plan select * from listtable where id >=3;
         """
-        containsAny("id#0 -> ndv=1.0000, min=3.000000(3), max=3.000000(3), count=1.0000,")
-        containsAny("id#0 -> ndv=1.0000, min=1.000000(1), max=3.000000(3), count=1.0000,")
+        contains("id#0 -> ndv=1.0000, min=3.000000(3), max=3.000000(3), count=1.0000")
     }
 }
 

From 845ed159dac37b70e73b86a507c3567727d81249 Mon Sep 17 00:00:00 2001
From: minghong <zhouminghong@selectdb.com>
Date: Thu, 9 Jan 2025 10:16:28 +0800
Subject: [PATCH 057/140] [fix](nereids) analyzing job should not be blocked by
 enable_stats=false (#46630)

### What problem does this PR solve?

 enable_stats is used to tell planner use column stats or not. It should not block analyze job
---
 .../main/java/org/apache/doris/analysis/AnalyzeTblStmt.java  | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java
index db98f6af706469e..feb3955ea93b9b3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java
@@ -120,12 +120,7 @@ public AnalyzeTblStmt(AnalyzeProperties analyzeProperties, TableName tableName,
     @Override
     @SuppressWarnings({"rawtypes"})
     public void analyze(Analyzer analyzer) throws UserException {
-        if (!ConnectContext.get().getSessionVariable().enableStats) {
-            throw new UserException("Analyze function is forbidden, you should add `enable_stats=true`"
-                    + "in your FE conf file");
-        }
         super.analyze(analyzer);
-
         tableName.analyze(analyzer);
 
         String catalogName = tableName.getCtl();

From 84d3867c0e58b20e5783efe42112e6e94532ca0f Mon Sep 17 00:00:00 2001
From: minghong <zhouminghong@selectdb.com>
Date: Thu, 9 Jan 2025 10:21:17 +0800
Subject: [PATCH 058/140] [fix](nereids)ignore partition row count if table row
 count is registerred (#46545)

### What problem does this PR solve?
this change only used for regression test purpose
---
 .../org/apache/doris/nereids/stats/StatsCalculator.java  | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
index e2909317ba1b946..64019818f5d1e74 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
@@ -449,6 +449,12 @@ private double getTableRowCount(CatalogRelation scan) {
         }
     }
 
+    private boolean isRegisteredRowCount(OlapScan olapScan) {
+        AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager();
+        TableStatsMeta tableMeta = analysisManager.findTableStatsStatus(olapScan.getTable().getId());
+        return tableMeta != null && tableMeta.userInjected;
+    }
+
     /**
      * if the table is not analyzed and BE does not report row count, return -1
      */
@@ -554,7 +560,8 @@ private Statistics computeOlapScan(OlapScan olapScan) {
             }
         }
 
-        if (olapScan.getSelectedPartitionIds().size() < olapScan.getTable().getPartitionNum()) {
+        if (!isRegisteredRowCount(olapScan)
+                && olapScan.getSelectedPartitionIds().size() < olapScan.getTable().getPartitionNum()) {
             // partition pruned
             // try to use selected partition stats, if failed, fall back to table stats
             double selectedPartitionsRowCount = getSelectedPartitionRowCount(olapScan, tableRowCount);

From 6396e8c2f7d61988e5822979b8bbdfd3fb42ce7a Mon Sep 17 00:00:00 2001
From: shuke <shuke@selectdb.com>
Date: Thu, 9 Jan 2025 11:10:29 +0800
Subject: [PATCH 059/140] [regression-test](case) fix manager case bug (#46659)

### What problem does this PR solve?

Issue Number: close #xxx

Related PR: #xxx

Problem Summary:

### Release note

None

### Check List (For Author)

- Test <!-- At least one of them must be included. -->
    - [x] Regression test
    - [ ] Unit Test
    - [ ] Manual test (add detailed scripts or steps below)
    - [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
        - [ ] Previous test can cover this change.
        - [ ] No code files have been changed.
        - [ ] Other reason <!-- Add your reason?  -->

- Behavior changed:
    - [x] No.
    - [ ] Yes. <!-- Explain the behavior change -->

- Does this need documentation?
    - [x] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->

### Check List (For Reviewer who merge this PR)

- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
 .../manager/test_manager_interface_1.groovy   | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/regression-test/suites/manager/test_manager_interface_1.groovy b/regression-test/suites/manager/test_manager_interface_1.groovy
index 63aba5f8294621b..41b7a7ae145694c 100644
--- a/regression-test/suites/manager/test_manager_interface_1.groovy
+++ b/regression-test/suites/manager/test_manager_interface_1.groovy
@@ -504,7 +504,7 @@ suite('test_manager_interface_1',"p0") {
         futures.add( thread {
             sleep(1500)
 
-            result = sql """ 
+            result = sql_return_maparray """ 
             select a.*, b.*, c.NAME as WORKLOAD_GROUP_NAME from information_schema.active_queries a left join 
             information_schema.backend_active_tasks b on a.QUERY_ID = b.QUERY_ID left join information_schema.workload_groups c on a.WORKLOAD_GROUP_ID = c.ID
             """
@@ -513,21 +513,21 @@ suite('test_manager_interface_1',"p0") {
             def x = 0
             def queryId = ""
             for( int i =0 ;i < result.size();i++ ){
-                assertTrue(result[i][0] != null ) // QueryId
+                assertTrue(result[i]["QUERY_ID"] != null ) // QueryId
 
-                if ( result[i][9].contains("sleep(4.7676)")  ){
+                if ( result[i]["SQL"].contains("sleep(4.7676)")  ){
                     x = 1 
-                    queryId = result[i][0]
+                    queryId = result[i]["QUERY_ID"]
                     logger.info("result = ${queryId}}")
 
-                    assertTrue(result[i][2]!=null) // QUERY_TIME_MS  
-                    assertTrue(result[i][14]!=null) // TASK_CPU_TIME_MS   
-                    assertTrue(result[i][15].toBigInteger() ==0 ) // SCAN_ROWS  
-                    assertTrue(result[i][16].toBigInteger() ==0)//SCAN_BYTES
-                    assertTrue(result[i][19].toBigInteger() ==0) // SHUFFLE_SEND_BYTES     
-                    assertTrue(result[i][20].toBigInteger() ==0) // SHUFFLE_SEND_ROWS   
-                    assertTrue(result[i][18]!=null) // CURRENT_USED_MEMORY_BYTES   
-                    assertTrue(result[i][22]!=null) // WORKLOAD_GROUP_NAME              
+                    assertTrue(result[i]["QUERY_TIME_MS"]!=null) // QUERY_TIME_MS  
+                    assertTrue(result[i]["TASK_CPU_TIME_MS"]!=null) // TASK_CPU_TIME_MS   
+                    assertTrue(result[i]["SCAN_ROWS"].toBigInteger() ==0 ) // SCAN_ROWS  
+                    assertTrue(result[i]["SCAN_BYTES"].toBigInteger() ==0)//SCAN_BYTES
+                    assertTrue(result[i]["SHUFFLE_SEND_BYTES"].toBigInteger() ==0) // SHUFFLE_SEND_BYTES     
+                    assertTrue(result[i]["SHUFFLE_SEND_ROWS"].toBigInteger() ==0) // SHUFFLE_SEND_ROWS   
+                    assertTrue(result[i]["CURRENT_USED_MEMORY_BYTES"]!=null) // CURRENT_USED_MEMORY_BYTES   
+                    assertTrue(result[i]["WORKLOAD_GROUP_ID"]!=null) // WORKLOAD_GROUP_NAME              
                 }
             }
             assertTrue(x == 1)

From cb4398593980bf26b6d77173acc96592633ada39 Mon Sep 17 00:00:00 2001
From: zfr95 <zhangfurong@selectdb.com>
Date: Thu, 9 Jan 2025 11:17:35 +0800
Subject: [PATCH 060/140] [test](mtmv) Check whether the constants in SQL will
 hit mtmv (#40442)

## Proposed changes

Add cases regarding whether MTMV normal rewriting is still supported
when constants exist in various query statements. There are various types of statements:
("plac_x" is a placeholder and will be replaced with various pre-prepared constants in the code.)
1. select plac_1 from tb
2. select col1 from (select plac_1 as col1 from tb) as t1
3. select plac_1 from tb group by plac_1
4. select plac_1 from tb where plac_1 >= 1 group by plac_1
5. select col1 from (select plac_1 as col1 from tb where plac_1 >= 1) as t1
6. select plac_1 from tb left join (select plac_1 as col1 from tb) as t1 on plac_1 = t1.col1
7. select plac_1 from tb1 left join (select plac_1 as col1 from tb2) as t1 on plac_1 = t1.col1 group by plac_1
8. select t1.col1 from (select plac_1 as col1 from tb1 where plac_1 = plac_1) as t1 left join (select plac_1 as col2 from tb2) as t2 on t1.col1 = t2.col2 group by t1.col1
9. select t1.col1 from (select plac_1 as col1 from tb1 where plac_1 = plac_1 group by col1) as t1 left join (select plac_1 as col2 from tb2) as t2 on t1.col1 = t2.col2 group by t1.col1
10. select plac_1 from tb1 left join (select plac_2 as col1 from tb2) as t1 on plac_1 = t1.col1
11. select plac_1 from tb1 left join (select plac_2 as col1 from tb2) as t1 on plac_1 = t1.col1 group by plac_1
12. select t1.col1 from (select plac_2 as col1 from tb1 where plac_2 = plac_2) as t1 left join (select plac_1 as col2 from tb2) as t2 on t1.col1 = t2.col2 group by t1.col1
13. select t1.col1 from (select plac_2 as col1 from tb2 where plac_2 = plac_2 group by col1) as t1 left join (select plac_1 as col2 from tb1) as t2 on t1.col1 = t2.col2 group by t1.col1
---
 .../constant_genaration_random_test_1.groovy  | 187 +++++++++++++++++
 .../constant_genaration_random_test_10.groovy | 186 +++++++++++++++++
 .../constant_genaration_random_test_11.groovy | 187 +++++++++++++++++
 .../constant_genaration_random_test_12.groovy | 187 +++++++++++++++++
 .../constant_genaration_random_test_13.groovy | 186 +++++++++++++++++
 .../constant_genaration_random_test_2.groovy  | 188 ++++++++++++++++++
 .../constant_genaration_random_test_3.groovy  | 186 +++++++++++++++++
 .../constant_genaration_random_test_4.groovy  | 187 +++++++++++++++++
 .../constant_genaration_random_test_5.groovy  | 187 +++++++++++++++++
 .../constant_genaration_random_test_6.groovy  | 186 +++++++++++++++++
 .../constant_genaration_random_test_7.groovy  | 187 +++++++++++++++++
 .../constant_genaration_random_test_8.groovy  | 187 +++++++++++++++++
 .../constant_genaration_random_test_9.groovy  | 187 +++++++++++++++++
 13 files changed, 2428 insertions(+)
 create mode 100644 regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_1.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_10.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_11.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_12.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_13.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_2.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_3.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_4.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_5.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_6.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_7.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_8.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_9.groovy

diff --git a/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_1.groovy b/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_1.groovy
new file mode 100644
index 000000000000000..ba2fb61ce820a8c
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_1.groovy
@@ -0,0 +1,187 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Test the case where a constant appears in the select list in basic SQL.
+// eg: select plac_1 from tb
+suite ("constant_genaration_random_mtmv_1", "constant_genaration_random_mtmv") {
+
+    String db = context.config.getDbNameByFile(context.file)
+    sql "use ${db}"
+    String table1 = "lineitem_constant_genaration_1"
+    String table2 = "orders_constant_genaration_1"
+    String mtmv_name = "constant_genaration_mtmv_1"
+
+    sql """
+    drop table if exists ${table2}
+    """
+
+    sql """CREATE TABLE `${table2}` (
+      `o_orderkey` BIGINT NULL,
+      `o_custkey` INT NULL,
+      `o_orderstatus` VARCHAR(1) NULL,
+      `o_totalprice` DECIMAL(15, 2)  NULL,
+      `o_orderpriority` VARCHAR(15) NULL,
+      `o_clerk` VARCHAR(15) NULL,
+      `o_shippriority` INT NULL,
+      `o_comment` VARCHAR(79) NULL,
+      `o_orderdate` DATE not NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(`o_orderkey`, `o_custkey`)
+    COMMENT 'OLAP'
+    auto partition by range (date_trunc(`o_orderdate`, 'day')) ()
+    DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1"
+    );"""
+
+    sql """
+    drop table if exists ${table1}
+    """
+
+    sql """CREATE TABLE `${table1}` (
+      `l_orderkey` BIGINT NULL,
+      `l_linenumber` INT NULL,
+      `l_partkey` INT NULL,
+      `l_suppkey` INT NULL,
+      `l_quantity` DECIMAL(15, 2) NULL,
+      `l_extendedprice` DECIMAL(15, 2) NULL,
+      `l_discount` DECIMAL(15, 2) NULL,
+      `l_tax` DECIMAL(15, 2) NULL,
+      `l_returnflag` VARCHAR(1) NULL,
+      `l_linestatus` VARCHAR(1) NULL,
+      `l_commitdate` DATE NULL,
+      `l_receiptdate` DATE NULL,
+      `l_shipinstruct` VARCHAR(25) NULL,
+      `l_shipmode` VARCHAR(10) NULL,
+      `l_comment` VARCHAR(44) NULL,
+      `l_shipdate` DATE not NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey )
+    COMMENT 'OLAP'
+    auto partition by range (date_trunc(`l_shipdate`, 'day')) ()
+    DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1"
+    );"""
+
+    sql """
+    insert into ${table2} values 
+    (null, 1, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
+    (0, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-19'),
+    (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-10-20'),
+    (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-10-21'),
+    (3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-10-22'),
+    (1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-19'),
+    (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-18'),
+    (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
+    (4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-19'); 
+    """
+
+    sql """
+    insert into ${table1} values 
+    (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (0, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (3, 3, null, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (1, 2, 3, null, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (2, 3, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-18'),
+    (3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17');
+    """
+
+    sql """analyze table ${table2} with sync;"""
+    sql """analyze table ${table1} with sync;"""
+
+    def create_mv = { mv_name, mv_sql ->
+        sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name};"""
+        sql """DROP TABLE IF EXISTS ${mv_name}"""
+        sql"""
+        CREATE MATERIALIZED VIEW ${mv_name} 
+        BUILD IMMEDIATE REFRESH AUTO ON MANUAL 
+        DISTRIBUTED BY RANDOM BUCKETS 2 
+        PROPERTIES ('replication_num' = '1') 
+        AS  
+        ${mv_sql}
+        """
+    }
+
+    def compare_res = { def stmt ->
+        sql "SET enable_materialized_view_rewrite=false"
+        def origin_res = sql stmt
+        logger.info("origin_res: " + origin_res)
+        sql "SET enable_materialized_view_rewrite=true"
+        def mv_origin_res = sql stmt
+        logger.info("mv_origin_res: " + mv_origin_res)
+        assertTrue((mv_origin_res == [] && origin_res == []) || (mv_origin_res.size() == origin_res.size()))
+        for (int row = 0; row < mv_origin_res.size(); row++) {
+            assertTrue(mv_origin_res[row].size() == origin_res[row].size())
+            for (int col = 0; col < mv_origin_res[row].size(); col++) {
+                assertTrue(mv_origin_res[row][col] == origin_res[row][col])
+            }
+        }
+    }
+
+    def param1 = "2"
+    def param2 = "l_orderkey"
+    def param3 = "abs(l_orderkey)"
+    def param4 = "2+l_orderkey"
+    def param5 = "2+abs(l_orderkey)"
+    def param6 = "l_orderkey + abs(l_orderkey)"
+    def param_lists1 = [param1, param2, param3, param4, param5, param6]
+
+    def param7 = "2"
+    def param8 = "o_orderkey"
+    def param9 = "abs(o_orderkey)"
+    def param10 = "2+o_orderkey"
+    def param11 = "2+abs(o_orderkey)"
+    def param12 = "o_orderkey + abs(o_orderkey)"
+    def param_lists2 = [param7, param8, param9, param10, param11, param12]
+
+    def sql1 = "select plac_1 from ${table1}"
+
+    def sql_lists = [sql1]
+
+    def check_not_chose = { def str, def mv_name ->
+        def sql_explain = sql """explain ${str};"""
+        def mv_index_1 = sql_explain.toString().indexOf("MaterializedViewRewriteSuccessAndChose:")
+        def mv_index_2 = sql_explain.toString().indexOf("MaterializedViewRewriteFail:")
+        assert(mv_index_1 != -1)
+        assert(mv_index_2 != -1)
+        if (sql_explain.toString().substring(mv_index_1, mv_index_2).indexOf(mv_name) != -1) {
+            return true
+        }
+        return false
+    }
+
+    for (int i = 0; i < sql_lists.size(); i++) {
+        for (int j = 0; j < param_lists1.size(); j++) {
+            for (int k = 0; k < param_lists2.size(); k++) {
+                logger.info("i: " + i + ", j: " + j + ", k: " + k)
+                def str = sql_lists[i].replaceAll("plac_1", param_lists1[j]).replaceAll("plac_2", param_lists2[k]).replaceAll(" on 2 = ", " on l_orderkey = ")
+
+                create_mv(mtmv_name, str)
+                waitingMTMVTaskFinishedByMvName(mtmv_name)
+
+                check_not_chose(str, mtmv_name)
+                compare_res(str + " order by 1")
+            }
+        }
+    }
+
+}
diff --git a/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_10.groovy b/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_10.groovy
new file mode 100644
index 000000000000000..9ea7a32cf09ec80
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_10.groovy
@@ -0,0 +1,186 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Test the case where multiple different constants appear in an SQL statement containing subqueries and the JOIN operator.
+// eg: select plac_1 from tb1 left join (select plac_2 as col1 from tb2) as t1 on plac_1 = t1.col1
+suite ("constant_genaration_random_mtmv_10", "constant_genaration_random_mtmv") {
+
+    String db = context.config.getDbNameByFile(context.file)
+    sql "use ${db}"
+    String table1 = "lineitem_constant_genaration_10"
+    String table2 = "orders_constant_genaration_10"
+    String mtmv_name = "constant_genaration_mtmv_10"
+
+    sql """
+    drop table if exists ${table2}
+    """
+
+    sql """CREATE TABLE `${table2}` (
+      `o_orderkey` BIGINT NULL,
+      `o_custkey` INT NULL,
+      `o_orderstatus` VARCHAR(1) NULL,
+      `o_totalprice` DECIMAL(15, 2)  NULL,
+      `o_orderpriority` VARCHAR(15) NULL,
+      `o_clerk` VARCHAR(15) NULL,
+      `o_shippriority` INT NULL,
+      `o_comment` VARCHAR(79) NULL,
+      `o_orderdate` DATE not NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(`o_orderkey`, `o_custkey`)
+    COMMENT 'OLAP'
+    auto partition by range (date_trunc(`o_orderdate`, 'day')) ()
+    DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1"
+    );"""
+
+    sql """
+    drop table if exists ${table1}
+    """
+
+    sql """CREATE TABLE `${table1}` (
+      `l_orderkey` BIGINT NULL,
+      `l_linenumber` INT NULL,
+      `l_partkey` INT NULL,
+      `l_suppkey` INT NULL,
+      `l_quantity` DECIMAL(15, 2) NULL,
+      `l_extendedprice` DECIMAL(15, 2) NULL,
+      `l_discount` DECIMAL(15, 2) NULL,
+      `l_tax` DECIMAL(15, 2) NULL,
+      `l_returnflag` VARCHAR(1) NULL,
+      `l_linestatus` VARCHAR(1) NULL,
+      `l_commitdate` DATE NULL,
+      `l_receiptdate` DATE NULL,
+      `l_shipinstruct` VARCHAR(25) NULL,
+      `l_shipmode` VARCHAR(10) NULL,
+      `l_comment` VARCHAR(44) NULL,
+      `l_shipdate` DATE not NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey )
+    COMMENT 'OLAP'
+    auto partition by range (date_trunc(`l_shipdate`, 'day')) ()
+    DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1"
+    );"""
+
+    sql """
+    insert into ${table2} values 
+    (null, 1, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
+    (0, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-19'),
+    (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-10-20'),
+    (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-10-21'),
+    (3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-10-22'),
+    (1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-19'),
+    (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-18'),
+    (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
+    (4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-19'); 
+    """
+
+    sql """
+    insert into ${table1} values 
+    (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (0, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (3, 3, null, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (1, 2, 3, null, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (2, 3, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-18'),
+    (3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17');
+    """
+
+    sql """analyze table ${table2} with sync;"""
+    sql """analyze table ${table1} with sync;"""
+
+    def create_mv = { mv_name, mv_sql ->
+        sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name};"""
+        sql """DROP TABLE IF EXISTS ${mv_name}"""
+        sql"""
+        CREATE MATERIALIZED VIEW ${mv_name} 
+        BUILD IMMEDIATE REFRESH AUTO ON MANUAL 
+        DISTRIBUTED BY RANDOM BUCKETS 2 
+        PROPERTIES ('replication_num' = '1') 
+        AS  
+        ${mv_sql}
+        """
+    }
+
+    def compare_res = { def stmt ->
+        sql "SET enable_materialized_view_rewrite=false"
+        def origin_res = sql stmt
+        logger.info("origin_res: " + origin_res)
+        sql "SET enable_materialized_view_rewrite=true"
+        def mv_origin_res = sql stmt
+        logger.info("mv_origin_res: " + mv_origin_res)
+        assertTrue((mv_origin_res == [] && origin_res == []) || (mv_origin_res.size() == origin_res.size()))
+        for (int row = 0; row < mv_origin_res.size(); row++) {
+            assertTrue(mv_origin_res[row].size() == origin_res[row].size())
+            for (int col = 0; col < mv_origin_res[row].size(); col++) {
+                assertTrue(mv_origin_res[row][col] == origin_res[row][col])
+            }
+        }
+    }
+
+    def param1 = "2"
+    def param2 = "l_orderkey"
+    def param3 = "abs(l_orderkey)"
+    def param4 = "2+l_orderkey"
+    def param5 = "2+abs(l_orderkey)"
+    def param6 = "l_orderkey + abs(l_orderkey)"
+    def param_lists1 = [param1, param2, param3, param4, param5, param6]
+
+    def param7 = "2"
+    def param8 = "o_orderkey"
+    def param9 = "abs(o_orderkey)"
+    def param10 = "2+o_orderkey"
+    def param11 = "2+abs(o_orderkey)"
+    def param12 = "o_orderkey + abs(o_orderkey)"
+    def param_lists2 = [param7, param8, param9, param10, param11, param12]
+
+    def sql12 = "select plac_1 from ${table1} left join (select plac_2 as col1 from ${table2}) as t1 on plac_1 = t1.col1 "
+    def sql_lists = [sql12]
+
+    def check_not_chose = { def str, def mv_name ->
+        def sql_explain = sql """explain ${str};"""
+        def mv_index_1 = sql_explain.toString().indexOf("MaterializedViewRewriteSuccessAndChose:")
+        def mv_index_2 = sql_explain.toString().indexOf("MaterializedViewRewriteFail:")
+        assert(mv_index_1 != -1)
+        assert(mv_index_2 != -1)
+        if (sql_explain.toString().substring(mv_index_1, mv_index_2).indexOf(mv_name) != -1) {
+            return true
+        }
+        return false
+    }
+
+    for (int i = 0; i < sql_lists.size(); i++) {
+        for (int j = 0; j < param_lists1.size(); j++) {
+            for (int k = 0; k < param_lists2.size(); k++) {
+                logger.info("i: " + i + ", j: " + j + ", k: " + k)
+                def str = sql_lists[i].replaceAll("plac_1", param_lists1[j]).replaceAll("plac_2", param_lists2[k]).replaceAll(" on 2 = ", " on l_orderkey = ")
+
+                create_mv(mtmv_name, str)
+                waitingMTMVTaskFinishedByMvName(mtmv_name)
+
+                check_not_chose(str, mtmv_name)
+                compare_res(str + " order by 1")
+            }
+        }
+    }
+    
+}
diff --git a/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_11.groovy b/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_11.groovy
new file mode 100644
index 000000000000000..83cad8259b94e6e
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_11.groovy
@@ -0,0 +1,187 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Test the case where multiple different constants appear in an SQL statement containing subqueries, the JOIN operator, and the aggregation operator.
+// eg: select plac_1 from tb1 left join (select plac_2 as col1 from tb2) as t1 on plac_1 = t1.col1 group by plac_1
+suite ("constant_genaration_random_mtmv_11", "constant_genaration_random_mtmv") {
+
+    String db = context.config.getDbNameByFile(context.file)
+    sql "use ${db}"
+    String table1 = "lineitem_constant_genaration_11"
+    String table2 = "orders_constant_genaration_11"
+    String mtmv_name = "constant_genaration_mtmv_11"
+
+    sql """
+    drop table if exists ${table2}
+    """
+
+    sql """CREATE TABLE `${table2}` (
+      `o_orderkey` BIGINT NULL,
+      `o_custkey` INT NULL,
+      `o_orderstatus` VARCHAR(1) NULL,
+      `o_totalprice` DECIMAL(15, 2)  NULL,
+      `o_orderpriority` VARCHAR(15) NULL,
+      `o_clerk` VARCHAR(15) NULL,
+      `o_shippriority` INT NULL,
+      `o_comment` VARCHAR(79) NULL,
+      `o_orderdate` DATE not NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(`o_orderkey`, `o_custkey`)
+    COMMENT 'OLAP'
+    auto partition by range (date_trunc(`o_orderdate`, 'day')) ()
+    DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1"
+    );"""
+
+    sql """
+    drop table if exists ${table1}
+    """
+
+    sql """CREATE TABLE `${table1}` (
+      `l_orderkey` BIGINT NULL,
+      `l_linenumber` INT NULL,
+      `l_partkey` INT NULL,
+      `l_suppkey` INT NULL,
+      `l_quantity` DECIMAL(15, 2) NULL,
+      `l_extendedprice` DECIMAL(15, 2) NULL,
+      `l_discount` DECIMAL(15, 2) NULL,
+      `l_tax` DECIMAL(15, 2) NULL,
+      `l_returnflag` VARCHAR(1) NULL,
+      `l_linestatus` VARCHAR(1) NULL,
+      `l_commitdate` DATE NULL,
+      `l_receiptdate` DATE NULL,
+      `l_shipinstruct` VARCHAR(25) NULL,
+      `l_shipmode` VARCHAR(10) NULL,
+      `l_comment` VARCHAR(44) NULL,
+      `l_shipdate` DATE not NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey )
+    COMMENT 'OLAP'
+    auto partition by range (date_trunc(`l_shipdate`, 'day')) ()
+    DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1"
+    );"""
+
+    sql """
+    insert into ${table2} values 
+    (null, 1, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
+    (0, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-19'),
+    (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-10-20'),
+    (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-10-21'),
+    (3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-10-22'),
+    (1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-19'),
+    (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-18'),
+    (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
+    (4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-19'); 
+    """
+
+    sql """
+    insert into ${table1} values 
+    (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (0, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (3, 3, null, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (1, 2, 3, null, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (2, 3, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-18'),
+    (3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17');
+    """
+
+    sql """analyze table ${table2} with sync;"""
+    sql """analyze table ${table1} with sync;"""
+
+    def create_mv = { mv_name, mv_sql ->
+        sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name};"""
+        sql """DROP TABLE IF EXISTS ${mv_name}"""
+        sql"""
+        CREATE MATERIALIZED VIEW ${mv_name} 
+        BUILD IMMEDIATE REFRESH AUTO ON MANUAL 
+        DISTRIBUTED BY RANDOM BUCKETS 2 
+        PROPERTIES ('replication_num' = '1') 
+        AS  
+        ${mv_sql}
+        """
+    }
+
+    def compare_res = { def stmt ->
+        sql "SET enable_materialized_view_rewrite=false"
+        def origin_res = sql stmt
+        logger.info("origin_res: " + origin_res)
+        sql "SET enable_materialized_view_rewrite=true"
+        def mv_origin_res = sql stmt
+        logger.info("mv_origin_res: " + mv_origin_res)
+        assertTrue((mv_origin_res == [] && origin_res == []) || (mv_origin_res.size() == origin_res.size()))
+        for (int row = 0; row < mv_origin_res.size(); row++) {
+            assertTrue(mv_origin_res[row].size() == origin_res[row].size())
+            for (int col = 0; col < mv_origin_res[row].size(); col++) {
+                assertTrue(mv_origin_res[row][col] == origin_res[row][col])
+            }
+        }
+    }
+
+    def param1 = "2"
+    def param2 = "l_orderkey"
+    def param3 = "abs(l_orderkey)"
+    def param4 = "2+l_orderkey"
+    def param5 = "2+abs(l_orderkey)"
+    def param6 = "l_orderkey + abs(l_orderkey)"
+    def param_lists1 = [param1, param2, param3, param4, param5, param6]
+
+    def param7 = "2"
+    def param8 = "o_orderkey"
+    def param9 = "abs(o_orderkey)"
+    def param10 = "2+o_orderkey"
+    def param11 = "2+abs(o_orderkey)"
+    def param12 = "o_orderkey + abs(o_orderkey)"
+    def param_lists2 = [param7, param8, param9, param10, param11, param12]
+
+    def sql13 = "select plac_1 from ${table1} left join (select plac_2 as col1 from ${table2}) as t1 on plac_1 = t1.col1 group by plac_1"
+
+    def sql_lists = [sql13]
+
+    def check_not_chose = { def str, def mv_name ->
+        def sql_explain = sql """explain ${str};"""
+        def mv_index_1 = sql_explain.toString().indexOf("MaterializedViewRewriteSuccessAndChose:")
+        def mv_index_2 = sql_explain.toString().indexOf("MaterializedViewRewriteFail:")
+        assert(mv_index_1 != -1)
+        assert(mv_index_2 != -1)
+        if (sql_explain.toString().substring(mv_index_1, mv_index_2).indexOf(mv_name) != -1) {
+            return true
+        }
+        return false
+    }
+
+    for (int i = 0; i < sql_lists.size(); i++) {
+        for (int j = 0; j < param_lists1.size(); j++) {
+            for (int k = 0; k < param_lists2.size(); k++) {
+                logger.info("i: " + i + ", j: " + j + ", k: " + k)
+                def str = sql_lists[i].replaceAll("plac_1", param_lists1[j]).replaceAll("plac_2", param_lists2[k]).replaceAll(" on 2 = ", " on l_orderkey = ")
+
+                create_mv(mtmv_name, str)
+                waitingMTMVTaskFinishedByMvName(mtmv_name)
+
+                check_not_chose(str, mtmv_name)
+                compare_res(str + " order by 1")
+            }
+        }
+    }
+    
+}
diff --git a/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_12.groovy b/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_12.groovy
new file mode 100644
index 000000000000000..ce8c5c817361fba
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_12.groovy
@@ -0,0 +1,187 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Test the case where multiple different constants appear in an SQL statement containing subqueries, the JOIN operator, and the aggregation operator.
+// eg: select t1.col1 from (select plac_2 as col1 from tb1 where plac_2 = plac_2) as t1 left join (select plac_1 as col2 from tb2) as t2 on t1.col1 = t2.col2 group by t1.col1
+suite ("constant_genaration_random_mtmv_12", "constant_genaration_random_mtmv") {
+
+    String db = context.config.getDbNameByFile(context.file)
+    sql "use ${db}"
+    String table1 = "lineitem_constant_genaration_12"
+    String table2 = "orders_constant_genaration_12"
+    String mtmv_name = "constant_genaration_mtmv_12"
+
+    sql """
+    drop table if exists ${table2}
+    """
+
+    sql """CREATE TABLE `${table2}` (
+      `o_orderkey` BIGINT NULL,
+      `o_custkey` INT NULL,
+      `o_orderstatus` VARCHAR(1) NULL,
+      `o_totalprice` DECIMAL(15, 2)  NULL,
+      `o_orderpriority` VARCHAR(15) NULL,
+      `o_clerk` VARCHAR(15) NULL,
+      `o_shippriority` INT NULL,
+      `o_comment` VARCHAR(79) NULL,
+      `o_orderdate` DATE not NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(`o_orderkey`, `o_custkey`)
+    COMMENT 'OLAP'
+    auto partition by range (date_trunc(`o_orderdate`, 'day')) ()
+    DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1"
+    );"""
+
+    sql """
+    drop table if exists ${table1}
+    """
+
+    sql """CREATE TABLE `${table1}` (
+      `l_orderkey` BIGINT NULL,
+      `l_linenumber` INT NULL,
+      `l_partkey` INT NULL,
+      `l_suppkey` INT NULL,
+      `l_quantity` DECIMAL(15, 2) NULL,
+      `l_extendedprice` DECIMAL(15, 2) NULL,
+      `l_discount` DECIMAL(15, 2) NULL,
+      `l_tax` DECIMAL(15, 2) NULL,
+      `l_returnflag` VARCHAR(1) NULL,
+      `l_linestatus` VARCHAR(1) NULL,
+      `l_commitdate` DATE NULL,
+      `l_receiptdate` DATE NULL,
+      `l_shipinstruct` VARCHAR(25) NULL,
+      `l_shipmode` VARCHAR(10) NULL,
+      `l_comment` VARCHAR(44) NULL,
+      `l_shipdate` DATE not NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey )
+    COMMENT 'OLAP'
+    auto partition by range (date_trunc(`l_shipdate`, 'day')) ()
+    DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1"
+    );"""
+
+    sql """
+    insert into ${table2} values 
+    (null, 1, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
+    (0, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-19'),
+    (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-10-20'),
+    (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-10-21'),
+    (3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-10-22'),
+    (1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-19'),
+    (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-18'),
+    (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
+    (4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-19'); 
+    """
+
+    sql """
+    insert into ${table1} values 
+    (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (0, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (3, 3, null, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (1, 2, 3, null, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (2, 3, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-18'),
+    (3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17');
+    """
+
+    sql """analyze table ${table2} with sync;"""
+    sql """analyze table ${table1} with sync;"""
+
+    def create_mv = { mv_name, mv_sql ->
+        sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name};"""
+        sql """DROP TABLE IF EXISTS ${mv_name}"""
+        sql"""
+        CREATE MATERIALIZED VIEW ${mv_name} 
+        BUILD IMMEDIATE REFRESH AUTO ON MANUAL 
+        DISTRIBUTED BY RANDOM BUCKETS 2 
+        PROPERTIES ('replication_num' = '1') 
+        AS  
+        ${mv_sql}
+        """
+    }
+
+    def compare_res = { def stmt ->
+        sql "SET enable_materialized_view_rewrite=false"
+        def origin_res = sql stmt
+        logger.info("origin_res: " + origin_res)
+        sql "SET enable_materialized_view_rewrite=true"
+        def mv_origin_res = sql stmt
+        logger.info("mv_origin_res: " + mv_origin_res)
+        assertTrue((mv_origin_res == [] && origin_res == []) || (mv_origin_res.size() == origin_res.size()))
+        for (int row = 0; row < mv_origin_res.size(); row++) {
+            assertTrue(mv_origin_res[row].size() == origin_res[row].size())
+            for (int col = 0; col < mv_origin_res[row].size(); col++) {
+                assertTrue(mv_origin_res[row][col] == origin_res[row][col])
+            }
+        }
+    }
+
+    def param1 = "2"
+    def param2 = "l_orderkey"
+    def param3 = "abs(l_orderkey)"
+    def param4 = "2+l_orderkey"
+    def param5 = "2+abs(l_orderkey)"
+    def param6 = "l_orderkey + abs(l_orderkey)"
+    def param_lists1 = [param1, param2, param3, param4, param5, param6]
+
+    def param7 = "2"
+    def param8 = "o_orderkey"
+    def param9 = "abs(o_orderkey)"
+    def param10 = "2+o_orderkey"
+    def param11 = "2+abs(o_orderkey)"
+    def param12 = "o_orderkey + abs(o_orderkey)"
+    def param_lists2 = [param7, param8, param9, param10, param11, param12]
+
+    def sql14 = "select t1.col1 from (select plac_2 as col1 from ${table2} where plac_2 = plac_2) as t1 left join (select plac_1 as col2 from ${table1}) as t2 on t1.col1 = t2.col2 group by t1.col1"
+
+    def sql_lists = [sql14]
+
+    def check_not_chose = { def str, def mv_name ->
+        def sql_explain = sql """explain ${str};"""
+        def mv_index_1 = sql_explain.toString().indexOf("MaterializedViewRewriteSuccessAndChose:")
+        def mv_index_2 = sql_explain.toString().indexOf("MaterializedViewRewriteFail:")
+        assert(mv_index_1 != -1)
+        assert(mv_index_2 != -1)
+        if (sql_explain.toString().substring(mv_index_1, mv_index_2).indexOf(mv_name) != -1) {
+            return true
+        }
+        return false
+    }
+
+    for (int i = 0; i < sql_lists.size(); i++) {
+        for (int j = 0; j < param_lists1.size(); j++) {
+            for (int k = 0; k < param_lists2.size(); k++) {
+                logger.info("i: " + i + ", j: " + j + ", k: " + k)
+                def str = sql_lists[i].replaceAll("plac_1", param_lists1[j]).replaceAll("plac_2", param_lists2[k]).replaceAll(" on 2 = ", " on l_orderkey = ")
+
+                create_mv(mtmv_name, str)
+                waitingMTMVTaskFinishedByMvName(mtmv_name)
+
+                check_not_chose(str, mtmv_name)
+                compare_res(str + " order by 1")
+            }
+        }
+    }
+    
+}
diff --git a/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_13.groovy b/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_13.groovy
new file mode 100644
index 000000000000000..9d4ba87fb9a3cf8
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_13.groovy
@@ -0,0 +1,186 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Test the case where multiple different constants appear in an SQL statement with subqueries, JOIN operator, FILTER operator, and aggregation operator.
+// eg: select t1.col1 from (select plac_2 as col1 from tb2 where plac_2 = plac_2 group by col1) as t1 left join (select plac_1 as col2 from tb1) as t2 on t1.col1 = t2.col2 group by t1.col1
+suite ("constant_genaration_random_mtmv_13", "constant_genaration_random_mtmv") {
+
+    String db = context.config.getDbNameByFile(context.file)
+    sql "use ${db}"
+    String table1 = "lineitem_constant_genaration_13"
+    String table2 = "orders_constant_genaration_13"
+    String mtmv_name = "constant_genaration_mtmv_13"
+
+    sql """
+    drop table if exists ${table2}
+    """
+
+    sql """CREATE TABLE `${table2}` (
+      `o_orderkey` BIGINT NULL,
+      `o_custkey` INT NULL,
+      `o_orderstatus` VARCHAR(1) NULL,
+      `o_totalprice` DECIMAL(15, 2)  NULL,
+      `o_orderpriority` VARCHAR(15) NULL,
+      `o_clerk` VARCHAR(15) NULL,
+      `o_shippriority` INT NULL,
+      `o_comment` VARCHAR(79) NULL,
+      `o_orderdate` DATE not NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(`o_orderkey`, `o_custkey`)
+    COMMENT 'OLAP'
+    auto partition by range (date_trunc(`o_orderdate`, 'day')) ()
+    DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1"
+    );"""
+
+    sql """
+    drop table if exists ${table1}
+    """
+
+    sql """CREATE TABLE `${table1}` (
+      `l_orderkey` BIGINT NULL,
+      `l_linenumber` INT NULL,
+      `l_partkey` INT NULL,
+      `l_suppkey` INT NULL,
+      `l_quantity` DECIMAL(15, 2) NULL,
+      `l_extendedprice` DECIMAL(15, 2) NULL,
+      `l_discount` DECIMAL(15, 2) NULL,
+      `l_tax` DECIMAL(15, 2) NULL,
+      `l_returnflag` VARCHAR(1) NULL,
+      `l_linestatus` VARCHAR(1) NULL,
+      `l_commitdate` DATE NULL,
+      `l_receiptdate` DATE NULL,
+      `l_shipinstruct` VARCHAR(25) NULL,
+      `l_shipmode` VARCHAR(10) NULL,
+      `l_comment` VARCHAR(44) NULL,
+      `l_shipdate` DATE not NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey )
+    COMMENT 'OLAP'
+    auto partition by range (date_trunc(`l_shipdate`, 'day')) ()
+    DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1"
+    );"""
+
+    sql """
+    insert into ${table2} values 
+    (null, 1, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
+    (0, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-19'),
+    (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-10-20'),
+    (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-10-21'),
+    (3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-10-22'),
+    (1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-19'),
+    (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-18'),
+    (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
+    (4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-19'); 
+    """
+
+    sql """
+    insert into ${table1} values 
+    (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (0, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (3, 3, null, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (1, 2, 3, null, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (2, 3, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-18'),
+    (3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17');
+    """
+
+    sql """analyze table ${table2} with sync;"""
+    sql """analyze table ${table1} with sync;"""
+
+    def create_mv = { mv_name, mv_sql ->
+        sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name};"""
+        sql """DROP TABLE IF EXISTS ${mv_name}"""
+        sql"""
+        CREATE MATERIALIZED VIEW ${mv_name} 
+        BUILD IMMEDIATE REFRESH AUTO ON MANUAL 
+        DISTRIBUTED BY RANDOM BUCKETS 2 
+        PROPERTIES ('replication_num' = '1') 
+        AS  
+        ${mv_sql}
+        """
+    }
+
+    def compare_res = { def stmt ->
+        sql "SET enable_materialized_view_rewrite=false"
+        def origin_res = sql stmt
+        logger.info("origin_res: " + origin_res)
+        sql "SET enable_materialized_view_rewrite=true"
+        def mv_origin_res = sql stmt
+        logger.info("mv_origin_res: " + mv_origin_res)
+        assertTrue((mv_origin_res == [] && origin_res == []) || (mv_origin_res.size() == origin_res.size()))
+        for (int row = 0; row < mv_origin_res.size(); row++) {
+            assertTrue(mv_origin_res[row].size() == origin_res[row].size())
+            for (int col = 0; col < mv_origin_res[row].size(); col++) {
+                assertTrue(mv_origin_res[row][col] == origin_res[row][col])
+            }
+        }
+    }
+
+    def param1 = "2"
+    def param2 = "l_orderkey"
+    def param3 = "abs(l_orderkey)"
+    def param4 = "2+l_orderkey"
+    def param5 = "2+abs(l_orderkey)"
+    def param6 = "l_orderkey + abs(l_orderkey)"
+    def param_lists1 = [param1, param2, param3, param4, param5, param6]
+
+    def param7 = "2"
+    def param8 = "o_orderkey"
+    def param9 = "abs(o_orderkey)"
+    def param10 = "2+o_orderkey"
+    def param11 = "2+abs(o_orderkey)"
+    def param12 = "o_orderkey + abs(o_orderkey)"
+    def param_lists2 = [param7, param8, param9, param10, param11, param12]
+
+    def sql15 = "select t1.col1 from (select plac_2 as col1 from ${table2} where plac_2 = plac_2 group by col1) as t1 left join (select plac_1 as col2 from ${table1}) as t2 on t1.col1 = t2.col2 group by t1.col1"
+    def sql_lists = [sql15]
+
+    def check_not_chose = { def str, def mv_name ->
+        def sql_explain = sql """explain ${str};"""
+        def mv_index_1 = sql_explain.toString().indexOf("MaterializedViewRewriteSuccessAndChose:")
+        def mv_index_2 = sql_explain.toString().indexOf("MaterializedViewRewriteFail:")
+        assert(mv_index_1 != -1)
+        assert(mv_index_2 != -1)
+        if (sql_explain.toString().substring(mv_index_1, mv_index_2).indexOf(mv_name) != -1) {
+            return true
+        }
+        return false
+    }
+
+    for (int i = 0; i < sql_lists.size(); i++) {
+        for (int j = 0; j < param_lists1.size(); j++) {
+            for (int k = 0; k < param_lists2.size(); k++) {
+                logger.info("i: " + i + ", j: " + j + ", k: " + k)
+                def str = sql_lists[i].replaceAll("plac_1", param_lists1[j]).replaceAll("plac_2", param_lists2[k]).replaceAll(" on 2 = ", " on l_orderkey = ")
+
+                create_mv(mtmv_name, str)
+                waitingMTMVTaskFinishedByMvName(mtmv_name)
+
+                check_not_chose(str, mtmv_name)
+                compare_res(str + " order by 1")
+            }
+        }
+    }
+    
+}
diff --git a/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_2.groovy b/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_2.groovy
new file mode 100644
index 000000000000000..8ddae0f7b238889
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_2.groovy
@@ -0,0 +1,188 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Test the case where a constant appears in the select list position of an SQL statement containing subqueries.
+// eg: select col1 from (select plac_1 as col1 from tb) as t1
+suite ("constant_genaration_random_mtmv_2", "constant_genaration_random_mtmv") {
+
+    String db = context.config.getDbNameByFile(context.file)
+    sql "use ${db}"
+    String table1 = "lineitem_constant_genaration_2"
+    String table2 = "orders_constant_genaration_2"
+    String mtmv_name = "constant_genaration_mtmv_2"
+
+    sql """
+    drop table if exists ${table2}
+    """
+
+    sql """CREATE TABLE `${table2}` (
+      `o_orderkey` BIGINT NULL,
+      `o_custkey` INT NULL,
+      `o_orderstatus` VARCHAR(1) NULL,
+      `o_totalprice` DECIMAL(15, 2)  NULL,
+      `o_orderpriority` VARCHAR(15) NULL,
+      `o_clerk` VARCHAR(15) NULL,
+      `o_shippriority` INT NULL,
+      `o_comment` VARCHAR(79) NULL,
+      `o_orderdate` DATE not NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(`o_orderkey`, `o_custkey`)
+    COMMENT 'OLAP'
+    auto partition by range (date_trunc(`o_orderdate`, 'day')) ()
+    DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1"
+    );"""
+
+    sql """
+    drop table if exists ${table1}
+    """
+
+    sql """CREATE TABLE `${table1}` (
+      `l_orderkey` BIGINT NULL,
+      `l_linenumber` INT NULL,
+      `l_partkey` INT NULL,
+      `l_suppkey` INT NULL,
+      `l_quantity` DECIMAL(15, 2) NULL,
+      `l_extendedprice` DECIMAL(15, 2) NULL,
+      `l_discount` DECIMAL(15, 2) NULL,
+      `l_tax` DECIMAL(15, 2) NULL,
+      `l_returnflag` VARCHAR(1) NULL,
+      `l_linestatus` VARCHAR(1) NULL,
+      `l_commitdate` DATE NULL,
+      `l_receiptdate` DATE NULL,
+      `l_shipinstruct` VARCHAR(25) NULL,
+      `l_shipmode` VARCHAR(10) NULL,
+      `l_comment` VARCHAR(44) NULL,
+      `l_shipdate` DATE not NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey )
+    COMMENT 'OLAP'
+    auto partition by range (date_trunc(`l_shipdate`, 'day')) ()
+    DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1"
+    );"""
+
+    sql """
+    insert into ${table2} values 
+    (null, 1, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
+    (0, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-19'),
+    (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-10-20'),
+    (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-10-21'),
+    (3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-10-22'),
+    (1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-19'),
+    (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-18'),
+    (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
+    (4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-19'); 
+    """
+
+    sql """
+    insert into ${table1} values 
+    (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (0, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (3, 3, null, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (1, 2, 3, null, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (2, 3, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-18'),
+    (3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17');
+    """
+
+    sql """analyze table ${table2} with sync;"""
+    sql """analyze table ${table1} with sync;"""
+
+    def create_mv = { mv_name, mv_sql ->
+        sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name};"""
+        sql """DROP TABLE IF EXISTS ${mv_name}"""
+        sql"""
+        CREATE MATERIALIZED VIEW ${mv_name} 
+        BUILD IMMEDIATE REFRESH AUTO ON MANUAL 
+        DISTRIBUTED BY RANDOM BUCKETS 2 
+        PROPERTIES ('replication_num' = '1') 
+        AS  
+        ${mv_sql}
+        """
+    }
+
+    def compare_res = { def stmt ->
+        sql "SET enable_materialized_view_rewrite=false"
+        def origin_res = sql stmt
+        logger.info("origin_res: " + origin_res)
+        sql "SET enable_materialized_view_rewrite=true"
+        def mv_origin_res = sql stmt
+        logger.info("mv_origin_res: " + mv_origin_res)
+        assertTrue((mv_origin_res == [] && origin_res == []) || (mv_origin_res.size() == origin_res.size()))
+        for (int row = 0; row < mv_origin_res.size(); row++) {
+            assertTrue(mv_origin_res[row].size() == origin_res[row].size())
+            for (int col = 0; col < mv_origin_res[row].size(); col++) {
+                assertTrue(mv_origin_res[row][col] == origin_res[row][col])
+            }
+        }
+    }
+
+    def param1 = "2"
+    def param2 = "l_orderkey"
+    def param3 = "abs(l_orderkey)"
+    def param4 = "2+l_orderkey"
+    def param5 = "2+abs(l_orderkey)"
+    def param6 = "l_orderkey + abs(l_orderkey)"
+    def param_lists1 = [param1, param2, param3, param4, param5, param6]
+
+    def param7 = "2"
+    def param8 = "o_orderkey"
+    def param9 = "abs(o_orderkey)"
+    def param10 = "2+o_orderkey"
+    def param11 = "2+abs(o_orderkey)"
+    def param12 = "o_orderkey + abs(o_orderkey)"
+    def param_lists2 = [param7, param8, param9, param10, param11, param12]
+
+
+    def sql3 = "select col1 from (select plac_1 as col1 from ${table1}) as t1"
+
+    def sql_lists = [sql3]
+
+    def check_not_chose = { def str, def mv_name ->
+        def sql_explain = sql """explain ${str};"""
+        def mv_index_1 = sql_explain.toString().indexOf("MaterializedViewRewriteSuccessAndChose:")
+        def mv_index_2 = sql_explain.toString().indexOf("MaterializedViewRewriteFail:")
+        assert(mv_index_1 != -1)
+        assert(mv_index_2 != -1)
+        if (sql_explain.toString().substring(mv_index_1, mv_index_2).indexOf(mv_name) != -1) {
+            return true
+        }
+        return false
+    }
+
+    for (int i = 0; i < sql_lists.size(); i++) {
+        for (int j = 0; j < param_lists1.size(); j++) {
+            for (int k = 0; k < param_lists2.size(); k++) {
+                logger.info("i: " + i + ", j: " + j + ", k: " + k)
+                def str = sql_lists[i].replaceAll("plac_1", param_lists1[j]).replaceAll("plac_2", param_lists2[k]).replaceAll(" on 2 = ", " on l_orderkey = ")
+
+                create_mv(mtmv_name, str)
+                waitingMTMVTaskFinishedByMvName(mtmv_name)
+
+                check_not_chose(str, mtmv_name)
+                compare_res(str + " order by 1")
+            }
+        }
+    }
+
+}
diff --git a/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_3.groovy b/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_3.groovy
new file mode 100644
index 000000000000000..3d5b521843bb33b
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_3.groovy
@@ -0,0 +1,186 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Test the case where a constant appears in the select list position of an SQL statement containing the GROUP BY operator.
+// eg: select plac_1 from tb group by plac_1
+suite ("constant_genaration_random_mtmv_3", "constant_genaration_random_mtmv") {
+
+    String db = context.config.getDbNameByFile(context.file)
+    sql "use ${db}"
+    String table1 = "lineitem_constant_genaration_3"
+    String table2 = "orders_constant_genaration_3"
+    String mtmv_name = "constant_genaration_mtmv_3"
+
+    sql """
+    drop table if exists ${table2}
+    """
+
+    sql """CREATE TABLE `${table2}` (
+      `o_orderkey` BIGINT NULL,
+      `o_custkey` INT NULL,
+      `o_orderstatus` VARCHAR(1) NULL,
+      `o_totalprice` DECIMAL(15, 2)  NULL,
+      `o_orderpriority` VARCHAR(15) NULL,
+      `o_clerk` VARCHAR(15) NULL,
+      `o_shippriority` INT NULL,
+      `o_comment` VARCHAR(79) NULL,
+      `o_orderdate` DATE not NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(`o_orderkey`, `o_custkey`)
+    COMMENT 'OLAP'
+    auto partition by range (date_trunc(`o_orderdate`, 'day')) ()
+    DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1"
+    );"""
+
+    sql """
+    drop table if exists ${table1}
+    """
+
+    sql """CREATE TABLE `${table1}` (
+      `l_orderkey` BIGINT NULL,
+      `l_linenumber` INT NULL,
+      `l_partkey` INT NULL,
+      `l_suppkey` INT NULL,
+      `l_quantity` DECIMAL(15, 2) NULL,
+      `l_extendedprice` DECIMAL(15, 2) NULL,
+      `l_discount` DECIMAL(15, 2) NULL,
+      `l_tax` DECIMAL(15, 2) NULL,
+      `l_returnflag` VARCHAR(1) NULL,
+      `l_linestatus` VARCHAR(1) NULL,
+      `l_commitdate` DATE NULL,
+      `l_receiptdate` DATE NULL,
+      `l_shipinstruct` VARCHAR(25) NULL,
+      `l_shipmode` VARCHAR(10) NULL,
+      `l_comment` VARCHAR(44) NULL,
+      `l_shipdate` DATE not NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey )
+    COMMENT 'OLAP'
+    auto partition by range (date_trunc(`l_shipdate`, 'day')) ()
+    DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1"
+    );"""
+
+    sql """
+    insert into ${table2} values 
+    (null, 1, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
+    (0, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-19'),
+    (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-10-20'),
+    (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-10-21'),
+    (3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-10-22'),
+    (1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-19'),
+    (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-18'),
+    (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
+    (4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-19'); 
+    """
+
+    sql """
+    insert into ${table1} values 
+    (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (0, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (3, 3, null, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (1, 2, 3, null, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (2, 3, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-18'),
+    (3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17');
+    """
+
+    sql """analyze table ${table2} with sync;"""
+    sql """analyze table ${table1} with sync;"""
+
+    def create_mv = { mv_name, mv_sql ->
+        sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name};"""
+        sql """DROP TABLE IF EXISTS ${mv_name}"""
+        sql"""
+        CREATE MATERIALIZED VIEW ${mv_name} 
+        BUILD IMMEDIATE REFRESH AUTO ON MANUAL 
+        DISTRIBUTED BY RANDOM BUCKETS 2 
+        PROPERTIES ('replication_num' = '1') 
+        AS  
+        ${mv_sql}
+        """
+    }
+
+    def compare_res = { def stmt ->
+        sql "SET enable_materialized_view_rewrite=false"
+        def origin_res = sql stmt
+        logger.info("origin_res: " + origin_res)
+        sql "SET enable_materialized_view_rewrite=true"
+        def mv_origin_res = sql stmt
+        logger.info("mv_origin_res: " + mv_origin_res)
+        assertTrue((mv_origin_res == [] && origin_res == []) || (mv_origin_res.size() == origin_res.size()))
+        for (int row = 0; row < mv_origin_res.size(); row++) {
+            assertTrue(mv_origin_res[row].size() == origin_res[row].size())
+            for (int col = 0; col < mv_origin_res[row].size(); col++) {
+                assertTrue(mv_origin_res[row][col] == origin_res[row][col])
+            }
+        }
+    }
+
+    def param1 = "2"
+    def param2 = "l_orderkey"
+    def param3 = "abs(l_orderkey)"
+    def param4 = "2+l_orderkey"
+    def param5 = "2+abs(l_orderkey)"
+    def param6 = "l_orderkey + abs(l_orderkey)"
+    def param_lists1 = [param1, param2, param3, param4, param5, param6]
+
+    def param7 = "2"
+    def param8 = "o_orderkey"
+    def param9 = "abs(o_orderkey)"
+    def param10 = "2+o_orderkey"
+    def param11 = "2+abs(o_orderkey)"
+    def param12 = "o_orderkey + abs(o_orderkey)"
+    def param_lists2 = [param7, param8, param9, param10, param11, param12]
+
+    def sql4 = "select plac_1 from ${table1} group by plac_1"
+    def sql_lists = [sql4]
+
+    def check_not_chose = { def str, def mv_name ->
+        def sql_explain = sql """explain ${str};"""
+        def mv_index_1 = sql_explain.toString().indexOf("MaterializedViewRewriteSuccessAndChose:")
+        def mv_index_2 = sql_explain.toString().indexOf("MaterializedViewRewriteFail:")
+        assert(mv_index_1 != -1)
+        assert(mv_index_2 != -1)
+        if (sql_explain.toString().substring(mv_index_1, mv_index_2).indexOf(mv_name) != -1) {
+            return true
+        }
+        return false
+    }
+
+    for (int i = 0; i < sql_lists.size(); i++) {
+        for (int j = 0; j < param_lists1.size(); j++) {
+            for (int k = 0; k < param_lists2.size(); k++) {
+                logger.info("i: " + i + ", j: " + j + ", k: " + k)
+                def str = sql_lists[i].replaceAll("plac_1", param_lists1[j]).replaceAll("plac_2", param_lists2[k]).replaceAll(" on 2 = ", " on l_orderkey = ")
+
+                create_mv(mtmv_name, str)
+                waitingMTMVTaskFinishedByMvName(mtmv_name)
+
+                check_not_chose(str, mtmv_name)
+                compare_res(str + " order by 1")
+            }
+        }
+    }
+
+}
diff --git a/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_4.groovy b/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_4.groovy
new file mode 100644
index 000000000000000..e79c2d767af1f7e
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_4.groovy
@@ -0,0 +1,187 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Test the case where a constant appears in the select list position of an SQL statement containing the GROUP BY operator and the FILTER operator.
+// eg: select plac_1 from tb where plac_1 >= 1 group by plac_1
+suite ("constant_genaration_random_mtmv_4", "constant_genaration_random_mtmv") {
+
+    String db = context.config.getDbNameByFile(context.file)
+    sql "use ${db}"
+    String table1 = "lineitem_constant_genaration_4"
+    String table2 = "orders_constant_genaration_4"
+    String mtmv_name = "constant_genaration_mtmv_4"
+
+    sql """
+    drop table if exists ${table2}
+    """
+
+    sql """CREATE TABLE `${table2}` (
+      `o_orderkey` BIGINT NULL,
+      `o_custkey` INT NULL,
+      `o_orderstatus` VARCHAR(1) NULL,
+      `o_totalprice` DECIMAL(15, 2)  NULL,
+      `o_orderpriority` VARCHAR(15) NULL,
+      `o_clerk` VARCHAR(15) NULL,
+      `o_shippriority` INT NULL,
+      `o_comment` VARCHAR(79) NULL,
+      `o_orderdate` DATE not NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(`o_orderkey`, `o_custkey`)
+    COMMENT 'OLAP'
+    auto partition by range (date_trunc(`o_orderdate`, 'day')) ()
+    DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1"
+    );"""
+
+    sql """
+    drop table if exists ${table1}
+    """
+
+    sql """CREATE TABLE `${table1}` (
+      `l_orderkey` BIGINT NULL,
+      `l_linenumber` INT NULL,
+      `l_partkey` INT NULL,
+      `l_suppkey` INT NULL,
+      `l_quantity` DECIMAL(15, 2) NULL,
+      `l_extendedprice` DECIMAL(15, 2) NULL,
+      `l_discount` DECIMAL(15, 2) NULL,
+      `l_tax` DECIMAL(15, 2) NULL,
+      `l_returnflag` VARCHAR(1) NULL,
+      `l_linestatus` VARCHAR(1) NULL,
+      `l_commitdate` DATE NULL,
+      `l_receiptdate` DATE NULL,
+      `l_shipinstruct` VARCHAR(25) NULL,
+      `l_shipmode` VARCHAR(10) NULL,
+      `l_comment` VARCHAR(44) NULL,
+      `l_shipdate` DATE not NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey )
+    COMMENT 'OLAP'
+    auto partition by range (date_trunc(`l_shipdate`, 'day')) ()
+    DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1"
+    );"""
+
+    sql """
+    insert into ${table2} values 
+    (null, 1, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
+    (0, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-19'),
+    (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-10-20'),
+    (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-10-21'),
+    (3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-10-22'),
+    (1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-19'),
+    (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-18'),
+    (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
+    (4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-19'); 
+    """
+
+    sql """
+    insert into ${table1} values 
+    (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (0, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (3, 3, null, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (1, 2, 3, null, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (2, 3, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-18'),
+    (3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17');
+    """
+
+    sql """analyze table ${table2} with sync;"""
+    sql """analyze table ${table1} with sync;"""
+
+    def create_mv = { mv_name, mv_sql ->
+        sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name};"""
+        sql """DROP TABLE IF EXISTS ${mv_name}"""
+        sql"""
+        CREATE MATERIALIZED VIEW ${mv_name} 
+        BUILD IMMEDIATE REFRESH AUTO ON MANUAL 
+        DISTRIBUTED BY RANDOM BUCKETS 2 
+        PROPERTIES ('replication_num' = '1') 
+        AS  
+        ${mv_sql}
+        """
+    }
+
+    def compare_res = { def stmt ->
+        sql "SET enable_materialized_view_rewrite=false"
+        def origin_res = sql stmt
+        logger.info("origin_res: " + origin_res)
+        sql "SET enable_materialized_view_rewrite=true"
+        def mv_origin_res = sql stmt
+        logger.info("mv_origin_res: " + mv_origin_res)
+        assertTrue((mv_origin_res == [] && origin_res == []) || (mv_origin_res.size() == origin_res.size()))
+        for (int row = 0; row < mv_origin_res.size(); row++) {
+            assertTrue(mv_origin_res[row].size() == origin_res[row].size())
+            for (int col = 0; col < mv_origin_res[row].size(); col++) {
+                assertTrue(mv_origin_res[row][col] == origin_res[row][col])
+            }
+        }
+    }
+
+    def param1 = "2"
+    def param2 = "l_orderkey"
+    def param3 = "abs(l_orderkey)"
+    def param4 = "2+l_orderkey"
+    def param5 = "2+abs(l_orderkey)"
+    def param6 = "l_orderkey + abs(l_orderkey)"
+    def param_lists1 = [param1, param2, param3, param4, param5, param6]
+
+    def param7 = "2"
+    def param8 = "o_orderkey"
+    def param9 = "abs(o_orderkey)"
+    def param10 = "2+o_orderkey"
+    def param11 = "2+abs(o_orderkey)"
+    def param12 = "o_orderkey + abs(o_orderkey)"
+    def param_lists2 = [param7, param8, param9, param10, param11, param12]
+
+    def sql5 = "select plac_1 from ${table1} where plac_1 >= 1 group by plac_1"
+
+    def sql_lists = [sql5]
+
+    def check_not_chose = { def str, def mv_name ->
+        def sql_explain = sql """explain ${str};"""
+        def mv_index_1 = sql_explain.toString().indexOf("MaterializedViewRewriteSuccessAndChose:")
+        def mv_index_2 = sql_explain.toString().indexOf("MaterializedViewRewriteFail:")
+        assert(mv_index_1 != -1)
+        assert(mv_index_2 != -1)
+        if (sql_explain.toString().substring(mv_index_1, mv_index_2).indexOf(mv_name) != -1) {
+            return true
+        }
+        return false
+    }
+
+    for (int i = 0; i < sql_lists.size(); i++) {
+        for (int j = 0; j < param_lists1.size(); j++) {
+            for (int k = 0; k < param_lists2.size(); k++) {
+                logger.info("i: " + i + ", j: " + j + ", k: " + k)
+                def str = sql_lists[i].replaceAll("plac_1", param_lists1[j]).replaceAll("plac_2", param_lists2[k]).replaceAll(" on 2 = ", " on l_orderkey = ")
+
+                create_mv(mtmv_name, str)
+                waitingMTMVTaskFinishedByMvName(mtmv_name)
+
+                check_not_chose(str, mtmv_name)
+                compare_res(str + " order by 1")
+            }
+        }
+    }
+
+}
diff --git a/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_5.groovy b/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_5.groovy
new file mode 100644
index 000000000000000..45d518af26ddce1
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_5.groovy
@@ -0,0 +1,187 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Test the case where a constant appears in the select list position of a subquery in an SQL statement that contains the GROUP BY operator and the FILTER operator.
+// eg: select col1 from (select plac_1 as col1 from tb where plac_1 >= 1) as t1
+suite ("constant_genaration_random_mtmv_5", "constant_genaration_random_mtmv") {
+
+    String db = context.config.getDbNameByFile(context.file)
+    sql "use ${db}"
+    String table1 = "lineitem_constant_genaration_5"
+    String table2 = "orders_constant_genaration_5"
+    String mtmv_name = "constant_genaration_mtmv_5"
+
+    sql """
+    drop table if exists ${table2}
+    """
+
+    sql """CREATE TABLE `${table2}` (
+      `o_orderkey` BIGINT NULL,
+      `o_custkey` INT NULL,
+      `o_orderstatus` VARCHAR(1) NULL,
+      `o_totalprice` DECIMAL(15, 2)  NULL,
+      `o_orderpriority` VARCHAR(15) NULL,
+      `o_clerk` VARCHAR(15) NULL,
+      `o_shippriority` INT NULL,
+      `o_comment` VARCHAR(79) NULL,
+      `o_orderdate` DATE not NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(`o_orderkey`, `o_custkey`)
+    COMMENT 'OLAP'
+    auto partition by range (date_trunc(`o_orderdate`, 'day')) ()
+    DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1"
+    );"""
+
+    sql """
+    drop table if exists ${table1}
+    """
+
+    sql """CREATE TABLE `${table1}` (
+      `l_orderkey` BIGINT NULL,
+      `l_linenumber` INT NULL,
+      `l_partkey` INT NULL,
+      `l_suppkey` INT NULL,
+      `l_quantity` DECIMAL(15, 2) NULL,
+      `l_extendedprice` DECIMAL(15, 2) NULL,
+      `l_discount` DECIMAL(15, 2) NULL,
+      `l_tax` DECIMAL(15, 2) NULL,
+      `l_returnflag` VARCHAR(1) NULL,
+      `l_linestatus` VARCHAR(1) NULL,
+      `l_commitdate` DATE NULL,
+      `l_receiptdate` DATE NULL,
+      `l_shipinstruct` VARCHAR(25) NULL,
+      `l_shipmode` VARCHAR(10) NULL,
+      `l_comment` VARCHAR(44) NULL,
+      `l_shipdate` DATE not NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey )
+    COMMENT 'OLAP'
+    auto partition by range (date_trunc(`l_shipdate`, 'day')) ()
+    DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1"
+    );"""
+
+    sql """
+    insert into ${table2} values 
+    (null, 1, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
+    (0, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-19'),
+    (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-10-20'),
+    (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-10-21'),
+    (3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-10-22'),
+    (1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-19'),
+    (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-18'),
+    (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
+    (4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-19'); 
+    """
+
+    sql """
+    insert into ${table1} values 
+    (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (0, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (3, 3, null, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (1, 2, 3, null, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (2, 3, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-18'),
+    (3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17');
+    """
+
+    sql """analyze table ${table2} with sync;"""
+    sql """analyze table ${table1} with sync;"""
+
+    def create_mv = { mv_name, mv_sql ->
+        sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name};"""
+        sql """DROP TABLE IF EXISTS ${mv_name}"""
+        sql"""
+        CREATE MATERIALIZED VIEW ${mv_name} 
+        BUILD IMMEDIATE REFRESH AUTO ON MANUAL 
+        DISTRIBUTED BY RANDOM BUCKETS 2 
+        PROPERTIES ('replication_num' = '1') 
+        AS  
+        ${mv_sql}
+        """
+    }
+
+    def compare_res = { def stmt ->
+        sql "SET enable_materialized_view_rewrite=false"
+        def origin_res = sql stmt
+        logger.info("origin_res: " + origin_res)
+        sql "SET enable_materialized_view_rewrite=true"
+        def mv_origin_res = sql stmt
+        logger.info("mv_origin_res: " + mv_origin_res)
+        assertTrue((mv_origin_res == [] && origin_res == []) || (mv_origin_res.size() == origin_res.size()))
+        for (int row = 0; row < mv_origin_res.size(); row++) {
+            assertTrue(mv_origin_res[row].size() == origin_res[row].size())
+            for (int col = 0; col < mv_origin_res[row].size(); col++) {
+                assertTrue(mv_origin_res[row][col] == origin_res[row][col])
+            }
+        }
+    }
+
+    def param1 = "2"
+    def param2 = "l_orderkey"
+    def param3 = "abs(l_orderkey)"
+    def param4 = "2+l_orderkey"
+    def param5 = "2+abs(l_orderkey)"
+    def param6 = "l_orderkey + abs(l_orderkey)"
+    def param_lists1 = [param1, param2, param3, param4, param5, param6]
+
+    def param7 = "2"
+    def param8 = "o_orderkey"
+    def param9 = "abs(o_orderkey)"
+    def param10 = "2+o_orderkey"
+    def param11 = "2+abs(o_orderkey)"
+    def param12 = "o_orderkey + abs(o_orderkey)"
+    def param_lists2 = [param7, param8, param9, param10, param11, param12]
+
+    def sql7 = "select col1 from (select plac_1 as col1 from ${table1} where plac_1 >= 1) as t1"
+
+    def sql_lists = [sql7]
+
+    def check_not_chose = { def str, def mv_name ->
+        def sql_explain = sql """explain ${str};"""
+        def mv_index_1 = sql_explain.toString().indexOf("MaterializedViewRewriteSuccessAndChose:")
+        def mv_index_2 = sql_explain.toString().indexOf("MaterializedViewRewriteFail:")
+        assert(mv_index_1 != -1)
+        assert(mv_index_2 != -1)
+        if (sql_explain.toString().substring(mv_index_1, mv_index_2).indexOf(mv_name) != -1) {
+            return true
+        }
+        return false
+    }
+
+    for (int i = 0; i < sql_lists.size(); i++) {
+        for (int j = 0; j < param_lists1.size(); j++) {
+            for (int k = 0; k < param_lists2.size(); k++) {
+                logger.info("i: " + i + ", j: " + j + ", k: " + k)
+                def str = sql_lists[i].replaceAll("plac_1", param_lists1[j]).replaceAll("plac_2", param_lists2[k]).replaceAll(" on 2 = ", " on l_orderkey = ")
+
+                create_mv(mtmv_name, str)
+                waitingMTMVTaskFinishedByMvName(mtmv_name)
+
+                check_not_chose(str, mtmv_name)
+                compare_res(str + " order by 1")
+            }
+        }
+    }
+
+}
diff --git a/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_6.groovy b/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_6.groovy
new file mode 100644
index 000000000000000..48a6015d40c2cda
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_6.groovy
@@ -0,0 +1,186 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Test the case where a constant appears in the select list position of an SQL statement containing the JOIN operator.
+// eg: select plac_1 from tb left join (select plac_1 as col1 from tb) as t1 on plac_1 = t1.col1
+suite ("constant_genaration_random_mtmv_6", "constant_genaration_random_mtmv") {
+
+    String db = context.config.getDbNameByFile(context.file)
+    sql "use ${db}"
+    String table1 = "lineitem_constant_genaration_6"
+    String table2 = "orders_constant_genaration_6"
+    String mtmv_name = "constant_genaration_mtmv_6"
+
+    sql """
+    drop table if exists ${table2}
+    """
+
+    sql """CREATE TABLE `${table2}` (
+      `o_orderkey` BIGINT NULL,
+      `o_custkey` INT NULL,
+      `o_orderstatus` VARCHAR(1) NULL,
+      `o_totalprice` DECIMAL(15, 2)  NULL,
+      `o_orderpriority` VARCHAR(15) NULL,
+      `o_clerk` VARCHAR(15) NULL,
+      `o_shippriority` INT NULL,
+      `o_comment` VARCHAR(79) NULL,
+      `o_orderdate` DATE not NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(`o_orderkey`, `o_custkey`)
+    COMMENT 'OLAP'
+    auto partition by range (date_trunc(`o_orderdate`, 'day')) ()
+    DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1"
+    );"""
+
+    sql """
+    drop table if exists ${table1}
+    """
+
+    sql """CREATE TABLE `${table1}` (
+      `l_orderkey` BIGINT NULL,
+      `l_linenumber` INT NULL,
+      `l_partkey` INT NULL,
+      `l_suppkey` INT NULL,
+      `l_quantity` DECIMAL(15, 2) NULL,
+      `l_extendedprice` DECIMAL(15, 2) NULL,
+      `l_discount` DECIMAL(15, 2) NULL,
+      `l_tax` DECIMAL(15, 2) NULL,
+      `l_returnflag` VARCHAR(1) NULL,
+      `l_linestatus` VARCHAR(1) NULL,
+      `l_commitdate` DATE NULL,
+      `l_receiptdate` DATE NULL,
+      `l_shipinstruct` VARCHAR(25) NULL,
+      `l_shipmode` VARCHAR(10) NULL,
+      `l_comment` VARCHAR(44) NULL,
+      `l_shipdate` DATE not NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey )
+    COMMENT 'OLAP'
+    auto partition by range (date_trunc(`l_shipdate`, 'day')) ()
+    DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1"
+    );"""
+
+    sql """
+    insert into ${table2} values 
+    (null, 1, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
+    (0, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-19'),
+    (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-10-20'),
+    (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-10-21'),
+    (3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-10-22'),
+    (1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-19'),
+    (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-18'),
+    (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
+    (4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-19'); 
+    """
+
+    sql """
+    insert into ${table1} values 
+    (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (0, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (3, 3, null, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (1, 2, 3, null, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (2, 3, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-18'),
+    (3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17');
+    """
+
+    sql """analyze table ${table2} with sync;"""
+    sql """analyze table ${table1} with sync;"""
+
+    def create_mv = { mv_name, mv_sql ->
+        sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name};"""
+        sql """DROP TABLE IF EXISTS ${mv_name}"""
+        sql"""
+        CREATE MATERIALIZED VIEW ${mv_name} 
+        BUILD IMMEDIATE REFRESH AUTO ON MANUAL 
+        DISTRIBUTED BY RANDOM BUCKETS 2 
+        PROPERTIES ('replication_num' = '1') 
+        AS  
+        ${mv_sql}
+        """
+    }
+
+    def compare_res = { def stmt ->
+        sql "SET enable_materialized_view_rewrite=false"
+        def origin_res = sql stmt
+        logger.info("origin_res: " + origin_res)
+        sql "SET enable_materialized_view_rewrite=true"
+        def mv_origin_res = sql stmt
+        logger.info("mv_origin_res: " + mv_origin_res)
+        assertTrue((mv_origin_res == [] && origin_res == []) || (mv_origin_res.size() == origin_res.size()))
+        for (int row = 0; row < mv_origin_res.size(); row++) {
+            assertTrue(mv_origin_res[row].size() == origin_res[row].size())
+            for (int col = 0; col < mv_origin_res[row].size(); col++) {
+                assertTrue(mv_origin_res[row][col] == origin_res[row][col])
+            }
+        }
+    }
+
+    def param1 = "2"
+    def param2 = "l_orderkey"
+    def param3 = "abs(l_orderkey)"
+    def param4 = "2+l_orderkey"
+    def param5 = "2+abs(l_orderkey)"
+    def param6 = "l_orderkey + abs(l_orderkey)"
+    def param_lists1 = [param1, param2, param3, param4, param5, param6]
+
+    def param7 = "2"
+    def param8 = "o_orderkey"
+    def param9 = "abs(o_orderkey)"
+    def param10 = "2+o_orderkey"
+    def param11 = "2+abs(o_orderkey)"
+    def param12 = "o_orderkey + abs(o_orderkey)"
+    def param_lists2 = [param7, param8, param9, param10, param11, param12]
+
+    def sql8 = "select plac_1 from ${table1} left join (select plac_1 as col1 from ${table1}) as t1 on plac_1 = t1.col1 "
+    def sql_lists = [sql8]
+
+    def check_not_chose = { def str, def mv_name ->
+        def sql_explain = sql """explain ${str};"""
+        def mv_index_1 = sql_explain.toString().indexOf("MaterializedViewRewriteSuccessAndChose:")
+        def mv_index_2 = sql_explain.toString().indexOf("MaterializedViewRewriteFail:")
+        assert(mv_index_1 != -1)
+        assert(mv_index_2 != -1)
+        if (sql_explain.toString().substring(mv_index_1, mv_index_2).indexOf(mv_name) != -1) {
+            return true
+        }
+        return false
+    }
+
+    for (int i = 0; i < sql_lists.size(); i++) {
+        for (int j = 0; j < param_lists1.size(); j++) {
+            for (int k = 0; k < param_lists2.size(); k++) {
+                logger.info("i: " + i + ", j: " + j + ", k: " + k)
+                def str = sql_lists[i].replaceAll("plac_1", param_lists1[j]).replaceAll("plac_2", param_lists2[k]).replaceAll(" on 2 = ", " on l_orderkey = ")
+
+                create_mv(mtmv_name, str)
+                waitingMTMVTaskFinishedByMvName(mtmv_name)
+
+                check_not_chose(str, mtmv_name)
+                compare_res(str + " order by 1")
+            }
+        }
+    }
+    
+}
diff --git a/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_7.groovy b/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_7.groovy
new file mode 100644
index 000000000000000..81ea82eae4e045a
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_7.groovy
@@ -0,0 +1,187 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Test the case where constants appear in an SQL statement containing the JOIN operator and subqueries.
+// eg: select plac_1 from tb1 left join (select plac_1 as col1 from tb2) as t1 on plac_1 = t1.col1 group by plac_1
+suite ("constant_genaration_random_mtmv_7", "constant_genaration_random_mtmv") {
+
+    String db = context.config.getDbNameByFile(context.file)
+    sql "use ${db}"
+    String table1 = "lineitem_constant_genaration_7"
+    String table2 = "orders_constant_genaration_7"
+    String mtmv_name = "constant_genaration_mtmv_7"
+
+    sql """
+    drop table if exists ${table2}
+    """
+
+    sql """CREATE TABLE `${table2}` (
+      `o_orderkey` BIGINT NULL,
+      `o_custkey` INT NULL,
+      `o_orderstatus` VARCHAR(1) NULL,
+      `o_totalprice` DECIMAL(15, 2)  NULL,
+      `o_orderpriority` VARCHAR(15) NULL,
+      `o_clerk` VARCHAR(15) NULL,
+      `o_shippriority` INT NULL,
+      `o_comment` VARCHAR(79) NULL,
+      `o_orderdate` DATE not NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(`o_orderkey`, `o_custkey`)
+    COMMENT 'OLAP'
+    auto partition by range (date_trunc(`o_orderdate`, 'day')) ()
+    DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1"
+    );"""
+
+    sql """
+    drop table if exists ${table1}
+    """
+
+    sql """CREATE TABLE `${table1}` (
+      `l_orderkey` BIGINT NULL,
+      `l_linenumber` INT NULL,
+      `l_partkey` INT NULL,
+      `l_suppkey` INT NULL,
+      `l_quantity` DECIMAL(15, 2) NULL,
+      `l_extendedprice` DECIMAL(15, 2) NULL,
+      `l_discount` DECIMAL(15, 2) NULL,
+      `l_tax` DECIMAL(15, 2) NULL,
+      `l_returnflag` VARCHAR(1) NULL,
+      `l_linestatus` VARCHAR(1) NULL,
+      `l_commitdate` DATE NULL,
+      `l_receiptdate` DATE NULL,
+      `l_shipinstruct` VARCHAR(25) NULL,
+      `l_shipmode` VARCHAR(10) NULL,
+      `l_comment` VARCHAR(44) NULL,
+      `l_shipdate` DATE not NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey )
+    COMMENT 'OLAP'
+    auto partition by range (date_trunc(`l_shipdate`, 'day')) ()
+    DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1"
+    );"""
+
+    sql """
+    insert into ${table2} values 
+    (null, 1, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
+    (0, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-19'),
+    (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-10-20'),
+    (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-10-21'),
+    (3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-10-22'),
+    (1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-19'),
+    (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-18'),
+    (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
+    (4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-19'); 
+    """
+
+    sql """
+    insert into ${table1} values 
+    (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (0, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (3, 3, null, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (1, 2, 3, null, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (2, 3, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-18'),
+    (3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17');
+    """
+
+    sql """analyze table ${table2} with sync;"""
+    sql """analyze table ${table1} with sync;"""
+
+    def create_mv = { mv_name, mv_sql ->
+        sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name};"""
+        sql """DROP TABLE IF EXISTS ${mv_name}"""
+        sql"""
+        CREATE MATERIALIZED VIEW ${mv_name} 
+        BUILD IMMEDIATE REFRESH AUTO ON MANUAL 
+        DISTRIBUTED BY RANDOM BUCKETS 2 
+        PROPERTIES ('replication_num' = '1') 
+        AS  
+        ${mv_sql}
+        """
+    }
+
+    def compare_res = { def stmt ->
+        sql "SET enable_materialized_view_rewrite=false"
+        def origin_res = sql stmt
+        logger.info("origin_res: " + origin_res)
+        sql "SET enable_materialized_view_rewrite=true"
+        def mv_origin_res = sql stmt
+        logger.info("mv_origin_res: " + mv_origin_res)
+        assertTrue((mv_origin_res == [] && origin_res == []) || (mv_origin_res.size() == origin_res.size()))
+        for (int row = 0; row < mv_origin_res.size(); row++) {
+            assertTrue(mv_origin_res[row].size() == origin_res[row].size())
+            for (int col = 0; col < mv_origin_res[row].size(); col++) {
+                assertTrue(mv_origin_res[row][col] == origin_res[row][col])
+            }
+        }
+    }
+
+    def param1 = "2"
+    def param2 = "l_orderkey"
+    def param3 = "abs(l_orderkey)"
+    def param4 = "2+l_orderkey"
+    def param5 = "2+abs(l_orderkey)"
+    def param6 = "l_orderkey + abs(l_orderkey)"
+    def param_lists1 = [param1, param2, param3, param4, param5, param6]
+
+    def param7 = "2"
+    def param8 = "o_orderkey"
+    def param9 = "abs(o_orderkey)"
+    def param10 = "2+o_orderkey"
+    def param11 = "2+abs(o_orderkey)"
+    def param12 = "o_orderkey + abs(o_orderkey)"
+    def param_lists2 = [param7, param8, param9, param10, param11, param12]
+
+
+    def sql9 = "select plac_1 from ${table1} left join (select plac_1 as col1 from ${table1}) as t1 on plac_1 = t1.col1 group by plac_1"
+    def sql_lists = [sql9]
+
+    def check_not_chose = { def str, def mv_name ->
+        def sql_explain = sql """explain ${str};"""
+        def mv_index_1 = sql_explain.toString().indexOf("MaterializedViewRewriteSuccessAndChose:")
+        def mv_index_2 = sql_explain.toString().indexOf("MaterializedViewRewriteFail:")
+        assert(mv_index_1 != -1)
+        assert(mv_index_2 != -1)
+        if (sql_explain.toString().substring(mv_index_1, mv_index_2).indexOf(mv_name) != -1) {
+            return true
+        }
+        return false
+    }
+
+    for (int i = 0; i < sql_lists.size(); i++) {
+        for (int j = 0; j < param_lists1.size(); j++) {
+            for (int k = 0; k < param_lists2.size(); k++) {
+                logger.info("i: " + i + ", j: " + j + ", k: " + k)
+                def str = sql_lists[i].replaceAll("plac_1", param_lists1[j]).replaceAll("plac_2", param_lists2[k]).replaceAll(" on 2 = ", " on l_orderkey = ")
+
+                create_mv(mtmv_name, str)
+                waitingMTMVTaskFinishedByMvName(mtmv_name)
+
+                check_not_chose(str, mtmv_name)
+                compare_res(str + " order by 1")
+            }
+        }
+    }
+    
+}
diff --git a/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_8.groovy b/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_8.groovy
new file mode 100644
index 000000000000000..042c3c4b334a3b8
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_8.groovy
@@ -0,0 +1,187 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Test the case where constants appear in an SQL statement for joining two subqueries.
+// eg: select t1.col1 from (select plac_1 as col1 from tb1 where plac_1 = plac_1) as t1 left join (select plac_1 as col2 from tb2) as t2 on t1.col1 = t2.col2 group by t1.col1
+suite ("constant_genaration_random_mtmv_8", "constant_genaration_random_mtmv") {
+
+    String db = context.config.getDbNameByFile(context.file)
+    sql "use ${db}"
+    String table1 = "lineitem_constant_genaration_8"
+    String table2 = "orders_constant_genaration_8"
+    String mtmv_name = "constant_genaration_mtmv_8"
+
+    sql """
+    drop table if exists ${table2}
+    """
+
+    sql """CREATE TABLE `${table2}` (
+      `o_orderkey` BIGINT NULL,
+      `o_custkey` INT NULL,
+      `o_orderstatus` VARCHAR(1) NULL,
+      `o_totalprice` DECIMAL(15, 2)  NULL,
+      `o_orderpriority` VARCHAR(15) NULL,
+      `o_clerk` VARCHAR(15) NULL,
+      `o_shippriority` INT NULL,
+      `o_comment` VARCHAR(79) NULL,
+      `o_orderdate` DATE not NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(`o_orderkey`, `o_custkey`)
+    COMMENT 'OLAP'
+    auto partition by range (date_trunc(`o_orderdate`, 'day')) ()
+    DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1"
+    );"""
+
+    sql """
+    drop table if exists ${table1}
+    """
+
+    sql """CREATE TABLE `${table1}` (
+      `l_orderkey` BIGINT NULL,
+      `l_linenumber` INT NULL,
+      `l_partkey` INT NULL,
+      `l_suppkey` INT NULL,
+      `l_quantity` DECIMAL(15, 2) NULL,
+      `l_extendedprice` DECIMAL(15, 2) NULL,
+      `l_discount` DECIMAL(15, 2) NULL,
+      `l_tax` DECIMAL(15, 2) NULL,
+      `l_returnflag` VARCHAR(1) NULL,
+      `l_linestatus` VARCHAR(1) NULL,
+      `l_commitdate` DATE NULL,
+      `l_receiptdate` DATE NULL,
+      `l_shipinstruct` VARCHAR(25) NULL,
+      `l_shipmode` VARCHAR(10) NULL,
+      `l_comment` VARCHAR(44) NULL,
+      `l_shipdate` DATE not NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey )
+    COMMENT 'OLAP'
+    auto partition by range (date_trunc(`l_shipdate`, 'day')) ()
+    DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1"
+    );"""
+
+    sql """
+    insert into ${table2} values 
+    (null, 1, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
+    (0, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-19'),
+    (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-10-20'),
+    (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-10-21'),
+    (3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-10-22'),
+    (1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-19'),
+    (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-18'),
+    (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
+    (4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-19'); 
+    """
+
+    sql """
+    insert into ${table1} values 
+    (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (0, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (3, 3, null, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (1, 2, 3, null, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (2, 3, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-18'),
+    (3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17');
+    """
+
+    sql """analyze table ${table2} with sync;"""
+    sql """analyze table ${table1} with sync;"""
+
+    def create_mv = { mv_name, mv_sql ->
+        sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name};"""
+        sql """DROP TABLE IF EXISTS ${mv_name}"""
+        sql"""
+        CREATE MATERIALIZED VIEW ${mv_name} 
+        BUILD IMMEDIATE REFRESH AUTO ON MANUAL 
+        DISTRIBUTED BY RANDOM BUCKETS 2 
+        PROPERTIES ('replication_num' = '1') 
+        AS  
+        ${mv_sql}
+        """
+    }
+
+    def compare_res = { def stmt ->
+        sql "SET enable_materialized_view_rewrite=false"
+        def origin_res = sql stmt
+        logger.info("origin_res: " + origin_res)
+        sql "SET enable_materialized_view_rewrite=true"
+        def mv_origin_res = sql stmt
+        logger.info("mv_origin_res: " + mv_origin_res)
+        assertTrue((mv_origin_res == [] && origin_res == []) || (mv_origin_res.size() == origin_res.size()))
+        for (int row = 0; row < mv_origin_res.size(); row++) {
+            assertTrue(mv_origin_res[row].size() == origin_res[row].size())
+            for (int col = 0; col < mv_origin_res[row].size(); col++) {
+                assertTrue(mv_origin_res[row][col] == origin_res[row][col])
+            }
+        }
+    }
+
+    def param1 = "2"
+    def param2 = "l_orderkey"
+    def param3 = "abs(l_orderkey)"
+    def param4 = "2+l_orderkey"
+    def param5 = "2+abs(l_orderkey)"
+    def param6 = "l_orderkey + abs(l_orderkey)"
+    def param_lists1 = [param1, param2, param3, param4, param5, param6]
+
+    def param7 = "2"
+    def param8 = "o_orderkey"
+    def param9 = "abs(o_orderkey)"
+    def param10 = "2+o_orderkey"
+    def param11 = "2+abs(o_orderkey)"
+    def param12 = "o_orderkey + abs(o_orderkey)"
+    def param_lists2 = [param7, param8, param9, param10, param11, param12]
+
+    def sql10 = "select t1.col1 from (select plac_1 as col1 from ${table1} where plac_1 = plac_1) as t1 left join (select plac_1 as col2 from ${table1}) as t2 on t1.col1 = t2.col2 group by t1.col1"
+
+    def sql_lists = [sql10]
+
+    def check_not_chose = { def str, def mv_name ->
+        def sql_explain = sql """explain ${str};"""
+        def mv_index_1 = sql_explain.toString().indexOf("MaterializedViewRewriteSuccessAndChose:")
+        def mv_index_2 = sql_explain.toString().indexOf("MaterializedViewRewriteFail:")
+        assert(mv_index_1 != -1)
+        assert(mv_index_2 != -1)
+        if (sql_explain.toString().substring(mv_index_1, mv_index_2).indexOf(mv_name) != -1) {
+            return true
+        }
+        return false
+    }
+
+    for (int i = 0; i < sql_lists.size(); i++) {
+        for (int j = 0; j < param_lists1.size(); j++) {
+            for (int k = 0; k < param_lists2.size(); k++) {
+                logger.info("i: " + i + ", j: " + j + ", k: " + k)
+                def str = sql_lists[i].replaceAll("plac_1", param_lists1[j]).replaceAll("plac_2", param_lists2[k]).replaceAll(" on 2 = ", " on l_orderkey = ")
+
+                create_mv(mtmv_name, str)
+                waitingMTMVTaskFinishedByMvName(mtmv_name)
+
+                check_not_chose(str, mtmv_name)
+                compare_res(str + " order by 1")
+            }
+        }
+    }
+
+}
diff --git a/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_9.groovy b/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_9.groovy
new file mode 100644
index 000000000000000..e6be7895afa0dd6
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/mv/genera_constant_sql/constant_genaration_random_test_9.groovy
@@ -0,0 +1,187 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Test the case where constants appear in an SQL statement that joins two subqueries with a GROUP BY clause.
+// eg: select t1.col1 from (select plac_1 as col1 from tb1 where plac_1 = plac_1 group by col1) as t1 left join (select plac_1 as col2 from tb2) as t2 on t1.col1 = t2.col2 group by t1.col1
+suite ("constant_genaration_random_mtmv_9", "constant_genaration_random_mtmv") {
+
+    String db = context.config.getDbNameByFile(context.file)
+    sql "use ${db}"
+    String table1 = "lineitem_constant_genaration_9"
+    String table2 = "orders_constant_genaration_9"
+    String mtmv_name = "constant_genaration_mtmv_9"
+
+    sql """
+    drop table if exists ${table2}
+    """
+
+    sql """CREATE TABLE `${table2}` (
+      `o_orderkey` BIGINT NULL,
+      `o_custkey` INT NULL,
+      `o_orderstatus` VARCHAR(1) NULL,
+      `o_totalprice` DECIMAL(15, 2)  NULL,
+      `o_orderpriority` VARCHAR(15) NULL,
+      `o_clerk` VARCHAR(15) NULL,
+      `o_shippriority` INT NULL,
+      `o_comment` VARCHAR(79) NULL,
+      `o_orderdate` DATE not NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(`o_orderkey`, `o_custkey`)
+    COMMENT 'OLAP'
+    auto partition by range (date_trunc(`o_orderdate`, 'day')) ()
+    DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1"
+    );"""
+
+    sql """
+    drop table if exists ${table1}
+    """
+
+    sql """CREATE TABLE `${table1}` (
+      `l_orderkey` BIGINT NULL,
+      `l_linenumber` INT NULL,
+      `l_partkey` INT NULL,
+      `l_suppkey` INT NULL,
+      `l_quantity` DECIMAL(15, 2) NULL,
+      `l_extendedprice` DECIMAL(15, 2) NULL,
+      `l_discount` DECIMAL(15, 2) NULL,
+      `l_tax` DECIMAL(15, 2) NULL,
+      `l_returnflag` VARCHAR(1) NULL,
+      `l_linestatus` VARCHAR(1) NULL,
+      `l_commitdate` DATE NULL,
+      `l_receiptdate` DATE NULL,
+      `l_shipinstruct` VARCHAR(25) NULL,
+      `l_shipmode` VARCHAR(10) NULL,
+      `l_comment` VARCHAR(44) NULL,
+      `l_shipdate` DATE not NULL
+    ) ENGINE=OLAP
+    DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey )
+    COMMENT 'OLAP'
+    auto partition by range (date_trunc(`l_shipdate`, 'day')) ()
+    DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1"
+    );"""
+
+    sql """
+    insert into ${table2} values 
+    (null, 1, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
+    (0, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'),
+    (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-19'),
+    (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-10-20'),
+    (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-10-21'),
+    (3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-10-22'),
+    (1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-19'),
+    (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-18'),
+    (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'),
+    (4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-19'); 
+    """
+
+    sql """
+    insert into ${table1} values 
+    (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (0, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (3, 3, null, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (1, 2, 3, null, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'),
+    (2, 3, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-18'),
+    (3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'),
+    (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17');
+    """
+
+    sql """analyze table ${table2} with sync;"""
+    sql """analyze table ${table1} with sync;"""
+
+    def create_mv = { mv_name, mv_sql ->
+        sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name};"""
+        sql """DROP TABLE IF EXISTS ${mv_name}"""
+        sql"""
+        CREATE MATERIALIZED VIEW ${mv_name} 
+        BUILD IMMEDIATE REFRESH AUTO ON MANUAL 
+        DISTRIBUTED BY RANDOM BUCKETS 2 
+        PROPERTIES ('replication_num' = '1') 
+        AS  
+        ${mv_sql}
+        """
+    }
+
+    def compare_res = { def stmt ->
+        sql "SET enable_materialized_view_rewrite=false"
+        def origin_res = sql stmt
+        logger.info("origin_res: " + origin_res)
+        sql "SET enable_materialized_view_rewrite=true"
+        def mv_origin_res = sql stmt
+        logger.info("mv_origin_res: " + mv_origin_res)
+        assertTrue((mv_origin_res == [] && origin_res == []) || (mv_origin_res.size() == origin_res.size()))
+        for (int row = 0; row < mv_origin_res.size(); row++) {
+            assertTrue(mv_origin_res[row].size() == origin_res[row].size())
+            for (int col = 0; col < mv_origin_res[row].size(); col++) {
+                assertTrue(mv_origin_res[row][col] == origin_res[row][col])
+            }
+        }
+    }
+
+    def param1 = "2"
+    def param2 = "l_orderkey"
+    def param3 = "abs(l_orderkey)"
+    def param4 = "2+l_orderkey"
+    def param5 = "2+abs(l_orderkey)"
+    def param6 = "l_orderkey + abs(l_orderkey)"
+    def param_lists1 = [param1, param2, param3, param4, param5, param6]
+
+    def param7 = "2"
+    def param8 = "o_orderkey"
+    def param9 = "abs(o_orderkey)"
+    def param10 = "2+o_orderkey"
+    def param11 = "2+abs(o_orderkey)"
+    def param12 = "o_orderkey + abs(o_orderkey)"
+    def param_lists2 = [param7, param8, param9, param10, param11, param12]
+
+    def sql11 = "select t1.col1 from (select plac_1 as col1 from ${table1} where plac_1 = plac_1 group by col1) as t1 left join (select plac_1 as col2 from ${table1}) as t2 on t1.col1 = t2.col2 group by t1.col1"
+
+    def sql_lists = [sql11]
+
+    def check_not_chose = { def str, def mv_name ->
+        def sql_explain = sql """explain ${str};"""
+        def mv_index_1 = sql_explain.toString().indexOf("MaterializedViewRewriteSuccessAndChose:")
+        def mv_index_2 = sql_explain.toString().indexOf("MaterializedViewRewriteFail:")
+        assert(mv_index_1 != -1)
+        assert(mv_index_2 != -1)
+        if (sql_explain.toString().substring(mv_index_1, mv_index_2).indexOf(mv_name) != -1) {
+            return true
+        }
+        return false
+    }
+
+    for (int i = 0; i < sql_lists.size(); i++) {
+        for (int j = 0; j < param_lists1.size(); j++) {
+            for (int k = 0; k < param_lists2.size(); k++) {
+                logger.info("i: " + i + ", j: " + j + ", k: " + k)
+                def str = sql_lists[i].replaceAll("plac_1", param_lists1[j]).replaceAll("plac_2", param_lists2[k]).replaceAll(" on 2 = ", " on l_orderkey = ")
+
+                create_mv(mtmv_name, str)
+                waitingMTMVTaskFinishedByMvName(mtmv_name)
+
+                check_not_chose(str, mtmv_name)
+                compare_res(str + " order by 1")
+            }
+        }
+    }
+
+}

From 9d28cf350505fe204ca92af58085e317647bb364 Mon Sep 17 00:00:00 2001
From: zy-kkk <zhongyk10@gmail.com>
Date: Thu, 9 Jan 2025 11:25:51 +0800
Subject: [PATCH 061/140] [fix](jdbc catalog) Use factory methods to return
 mapped types instead (#46623)

We should use factory methods to create types instead of returning a type directly, because the returned type is static and final, which will cause problems with length setting.
---
 .../apache/doris/datasource/jdbc/client/JdbcClient.java  | 4 ++--
 .../doris/datasource/jdbc/client/JdbcDB2Client.java      | 9 ++-------
 .../doris/datasource/jdbc/client/JdbcGbaseClient.java    | 6 +-----
 .../doris/datasource/jdbc/client/JdbcMySQLClient.java    | 6 ++----
 .../datasource/jdbc/client/JdbcPostgreSQLClient.java     | 5 +----
 .../doris/datasource/jdbc/client/JdbcSapHanaClient.java  | 5 +----
 .../doris/datasource/jdbc/client/JdbcTrinoClient.java    | 5 +----
 .../jdbc/type_test/tvf/test_mysql_all_types_tvf.out      | 8 ++++----
 8 files changed, 14 insertions(+), 34 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java
index f139dde5af2a2e1..a3dfdcda3193d4d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java
@@ -369,8 +369,8 @@ public List<JdbcFieldSchema> getJdbcColumnsInfo(String remoteDbName, String remo
         return tableSchema;
     }
 
-    public List<Column> getColumnsFromJdbc(String localDbName, String localTableName) {
-        List<JdbcFieldSchema> jdbcTableSchema = getJdbcColumnsInfo(localDbName, localTableName);
+    public List<Column> getColumnsFromJdbc(String remoteDbName, String remoteTableName) {
+        List<JdbcFieldSchema> jdbcTableSchema = getJdbcColumnsInfo(remoteDbName, remoteTableName);
         List<Column> dorisTableSchema = Lists.newArrayListWithCapacity(jdbcTableSchema.size());
         for (JdbcFieldSchema field : jdbcTableSchema) {
             dorisTableSchema.add(new Column(field.getColumnName(),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcDB2Client.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcDB2Client.java
index a353b7ac361353f..666b2b0f2a9ded9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcDB2Client.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcDB2Client.java
@@ -17,7 +17,6 @@
 
 package org.apache.doris.datasource.jdbc.client;
 
-import org.apache.doris.catalog.PrimitiveType;
 import org.apache.doris.catalog.ScalarType;
 import org.apache.doris.catalog.Type;
 import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
@@ -84,14 +83,10 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
             case "REAL":
                 return Type.FLOAT;
             case "CHAR":
-                ScalarType charType = ScalarType.createType(PrimitiveType.CHAR);
-                charType.setLength(fieldSchema.getColumnSize().orElse(0));
-                return charType;
+                return ScalarType.createCharType(fieldSchema.requiredColumnSize());
             case "VARCHAR":
             case "LONG VARCHAR":
-                ScalarType varcharType = ScalarType.createType(PrimitiveType.VARCHAR);
-                varcharType.setLength(fieldSchema.getColumnSize().orElse(0));
-                return varcharType;
+                return ScalarType.createVarcharType(fieldSchema.requiredColumnSize());
             case "DATE":
                 return ScalarType.createDateV2Type();
             case "TIMESTAMP": {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcGbaseClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcGbaseClient.java
index 086a8a5f393614b..6121ef2dbfc2fb3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcGbaseClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcGbaseClient.java
@@ -17,7 +17,6 @@
 
 package org.apache.doris.datasource.jdbc.client;
 
-import org.apache.doris.catalog.PrimitiveType;
 import org.apache.doris.catalog.ScalarType;
 import org.apache.doris.catalog.Type;
 import org.apache.doris.common.util.Util;
@@ -143,10 +142,7 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
                 return ScalarType.createDatetimeV2Type(scale);
             }
             case Types.CHAR:
-                ScalarType charType = ScalarType.createType(PrimitiveType.CHAR);
-                charType.setLength(fieldSchema.getColumnSize()
-                        .orElseThrow(() -> new IllegalArgumentException("Length not present")));
-                return charType;
+                return ScalarType.createCharType(fieldSchema.requiredColumnSize());
             case Types.TIME:
             case Types.VARCHAR:
             case Types.LONGVARCHAR:
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java
index f4c331cf35779c3..d69740a9269a8df 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java
@@ -257,11 +257,9 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
                 return createDecimalOrStringType(precision, scale);
             }
             case "CHAR":
-                ScalarType charType = ScalarType.createType(PrimitiveType.CHAR);
-                charType.setLength(fieldSchema.requiredColumnSize());
-                return charType;
+                return ScalarType.createCharType(fieldSchema.requiredColumnSize());
             case "VARCHAR":
-                return ScalarType.createVarcharType(fieldSchema.getColumnSize().orElse(0));
+                return ScalarType.createVarcharType(fieldSchema.requiredColumnSize());
             case "BIT":
                 if (fieldSchema.requiredColumnSize() == 1) {
                     return Type.BOOLEAN;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java
index 49e048939507266..21c16f2dd20ad54 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java
@@ -18,7 +18,6 @@
 package org.apache.doris.datasource.jdbc.client;
 
 import org.apache.doris.catalog.ArrayType;
-import org.apache.doris.catalog.PrimitiveType;
 import org.apache.doris.catalog.ScalarType;
 import org.apache.doris.catalog.Type;
 import org.apache.doris.common.util.Util;
@@ -115,9 +114,7 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
             case "float8":
                 return Type.DOUBLE;
             case "bpchar":
-                ScalarType charType = ScalarType.createType(PrimitiveType.CHAR);
-                charType.setLength(fieldSchema.getColumnSize().orElse(0));
-                return charType;
+                return ScalarType.createCharType(fieldSchema.requiredColumnSize());
             case "timestamp":
             case "timestamptz": {
                 // postgres can support microsecond
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSapHanaClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSapHanaClient.java
index f10d082bada3553..e89b6268b900f76 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSapHanaClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSapHanaClient.java
@@ -17,7 +17,6 @@
 
 package org.apache.doris.datasource.jdbc.client;
 
-import org.apache.doris.catalog.PrimitiveType;
 import org.apache.doris.catalog.ScalarType;
 import org.apache.doris.catalog.Type;
 import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
@@ -80,9 +79,7 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
                 return Type.BOOLEAN;
             case "CHAR":
             case "NCHAR":
-                ScalarType charType = ScalarType.createType(PrimitiveType.CHAR);
-                charType.setLength(fieldSchema.getColumnSize().orElse(0));
-                return charType;
+                return ScalarType.createCharType(fieldSchema.requiredColumnSize());
             case "TIME":
             case "VARCHAR":
             case "NVARCHAR":
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcTrinoClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcTrinoClient.java
index eac975805743c44..6c818a41cb6aca3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcTrinoClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcTrinoClient.java
@@ -18,7 +18,6 @@
 package org.apache.doris.datasource.jdbc.client;
 
 import org.apache.doris.catalog.ArrayType;
-import org.apache.doris.catalog.PrimitiveType;
 import org.apache.doris.catalog.ScalarType;
 import org.apache.doris.catalog.Type;
 import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
@@ -65,9 +64,7 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
         }
 
         if (trinoType.startsWith("char")) {
-            ScalarType charType = ScalarType.createType(PrimitiveType.CHAR);
-            charType.setLength(fieldSchema.getColumnSize().orElse(0));
-            return charType;
+            return ScalarType.createCharType(fieldSchema.requiredColumnSize());
         }
 
         if (trinoType.startsWith("timestamp")) {
diff --git a/regression-test/data/external_table_p0/jdbc/type_test/tvf/test_mysql_all_types_tvf.out b/regression-test/data/external_table_p0/jdbc/type_test/tvf/test_mysql_all_types_tvf.out
index dacd255426a9b0b..33f085b9550b44f 100644
--- a/regression-test/data/external_table_p0/jdbc/type_test/tvf/test_mysql_all_types_tvf.out
+++ b/regression-test/data/external_table_p0/jdbc/type_test/tvf/test_mysql_all_types_tvf.out
@@ -34,12 +34,12 @@ datetime	datetime	Yes	true	\N	NONE
 timestamp1	datetime	Yes	true	\N	NONE
 timestamp2	datetime(3)	Yes	true	\N	NONE
 timestamp3	datetime(6)	Yes	true	\N	NONE
-char	char(6)	Yes	true	\N	NONE
+char	char(5)	Yes	true	\N	NONE
 varchar	varchar(10)	Yes	true	\N	NONE
 text	text	Yes	true	\N	NONE
 blob	text	Yes	true	\N	NONE
 json	text	Yes	true	\N	NONE
-set	char(6)	Yes	true	\N	NONE
+set	char(23)	Yes	true	\N	NONE
 bit	text	Yes	true	\N	NONE
 binary	text	Yes	true	\N	NONE
 varbinary	text	Yes	true	\N	NONE
@@ -86,12 +86,12 @@ datetime	datetime	Yes	true	\N	NONE
 timestamp1	datetime	Yes	true	\N	NONE
 timestamp2	datetime(3)	Yes	true	\N	NONE
 timestamp3	datetime(6)	Yes	true	\N	NONE
-char	char(6)	Yes	true	\N	NONE
+char	char(5)	Yes	true	\N	NONE
 varchar	varchar(10)	Yes	true	\N	NONE
 text	text	Yes	true	\N	NONE
 blob	text	Yes	true	\N	NONE
 json	text	Yes	true	\N	NONE
-set	char(6)	Yes	true	\N	NONE
+set	char(23)	Yes	true	\N	NONE
 bit	text	Yes	true	\N	NONE
 binary	text	Yes	true	\N	NONE
 varbinary	text	Yes	true	\N	NONE

From 0c1570769edf3001fd5a1264b7851b8b7e99e8c9 Mon Sep 17 00:00:00 2001
From: Gabriel <liwenqiang@selectdb.com>
Date: Thu, 9 Jan 2025 11:31:33 +0800
Subject: [PATCH 062/140] [minor](conf) Deprecate variable
 `PARALLEL_FRAGMENT_EXEC_INSTANCE_NUM` (#46173)

---
 .../org/apache/doris/qe/SessionVariable.java  |  10 +-
 .../query_regression/result/issue_5171.result |   2 +-
 .../query_regression/result/issue_8778.result |   2 +-
 pytest/qe/query_regression/sql/issue_5171.sql |   2 +-
 pytest/qe/query_regression/sql/issue_8778.sql |   2 +-
 .../clickbench/conf/opt_session_variables.sql |   2 +-
 .../test_null_aware_left_anti_join.groovy     |   1 -
 .../test_scan_topn_limit.groovy               |   2 +-
 .../query/test_nested_type_with_resize.groovy |   8 +-
 .../hive/hive_tpch_sf1_orc.groovy             |  44 +++---
 .../hive/hive_tpch_sf1_parquet.groovy         |  44 +++---
 .../hive/test_trino_hive_tpch_sf1_orc.groovy  |  44 +++---
 .../test_trino_hive_tpch_sf1_parquet.groovy   |  44 +++---
 .../hive/test_external_github.groovy          |   8 +-
 .../nereids_hint_tpcds_p0/ddl/shape.tmpl      |   2 +-
 .../cte/test_cte_filter_pushdown.groovy       |   2 +-
 .../suites/nereids_p0/join/test_join.groovy   |   2 +-
 .../suites/nereids_p0/join/test_join5.groovy  |   2 +-
 .../nereids_p0/join/test_join_11.groovy       |   2 +-
 .../nereids_p0/join/test_join_12.groovy       |   2 +-
 .../nereids_p0/join/test_join_13.groovy       |   2 +-
 .../nereids_p0/join/test_join_14.groovy       |   2 +-
 .../nereids_p0/join/test_join_15.groovy       |   2 +-
 .../limit/sql/useRestrictiveLimitFromSubq.sql |   2 +-
 .../limit/sql/withGroupByInSubq.sql           |   2 +-
 .../nereids_p0/limit/sql/withJoinInSubq.sql   |   2 +-
 .../suites/nereids_p0/limit/sql/withSubq.sql  |   2 +-
 .../suites/nereids_p0/sort/sort.groovy        |   6 +-
 .../test_aggregate_retention.sql              |   2 +-
 .../test_bitmap_function.groovy               |   4 +-
 .../conditional_functions/test_nullif.groovy  |  10 +-
 .../test_date_function.groovy                 |   2 +-
 ...te_or_datetime_computation_negative.groovy |   2 +-
 .../test_window_function.groovy               | 128 +++++++++---------
 .../cte/test_cte_filter_pushdown.groovy       |   1 -
 .../mv/variant/variant_mv.groovy              |   2 +-
 .../group_by_constant.groovy                  |   2 +-
 .../null_aware_left_anti_join.groovy          |   2 +-
 .../nereids_syntax_p0/scan_partition.groovy   |   2 +-
 .../suites/nereids_tpch_p0/tpch/q1.groovy     |   2 +-
 .../suites/nereids_tpch_p0/tpch/q10.groovy    |   2 +-
 .../suites/nereids_tpch_p0/tpch/q11.groovy    |   2 +-
 .../suites/nereids_tpch_p0/tpch/q12.groovy    |   2 +-
 .../suites/nereids_tpch_p0/tpch/q13.groovy    |   2 +-
 .../suites/nereids_tpch_p0/tpch/q14.groovy    |   2 +-
 .../suites/nereids_tpch_p0/tpch/q15.groovy    |   2 +-
 .../suites/nereids_tpch_p0/tpch/q16.groovy    |   2 +-
 .../suites/nereids_tpch_p0/tpch/q17.groovy    |   2 +-
 .../suites/nereids_tpch_p0/tpch/q18.groovy    |   2 +-
 .../suites/nereids_tpch_p0/tpch/q19.groovy    |   2 +-
 .../suites/nereids_tpch_p0/tpch/q20.groovy    |   2 +-
 .../suites/nereids_tpch_p0/tpch/q21.groovy    |   2 +-
 .../suites/nereids_tpch_p0/tpch/q22.groovy    |   2 +-
 .../suites/nereids_tpch_p0/tpch/q3.groovy     |   2 +-
 .../suites/nereids_tpch_p0/tpch/q4.groovy     |   2 +-
 .../suites/nereids_tpch_p0/tpch/q6.groovy     |   2 +-
 .../suites/nereids_tpch_p0/tpch/q7.groovy     |   2 +-
 .../suites/nereids_tpch_p0/tpch/q8.groovy     |   2 +-
 .../suites/nereids_tpch_p0/tpch/q9.groovy     |   2 +-
 .../performance_p0/redundant_conjuncts.groovy |   2 +-
 ...oin_bucket_shffule_hash_partitioned.groovy |   4 +-
 .../join/test_join_result_count.groovy        |   2 +-
 .../join/test_partitioned_hash_join.groovy    |   2 +-
 .../test_runtime_filter_decimal256.groovy     |   2 +-
 .../test_invalid_session.groovy               |   6 -
 .../suites/query_p0/sort/sort.groovy          |   6 +-
 .../test_aggregate_retention.sql              |   2 +-
 .../test_bitmap_function.groovy               |   6 +-
 .../conditional_functions/test_nullif.groovy  |  10 +-
 .../test_query_limit.groovy                   |   6 +-
 .../test_date_function.groovy                 |   4 +-
 .../math_functions/test_conv.groovy           |   2 +-
 .../test_string_function.groovy               |   2 +-
 .../test_string_function_regexp.groovy        |   2 +-
 .../test_window_function.groovy               | 128 +++++++++---------
 .../shape_check/tpcds_sf100/ddl/rf_prune.tmpl |   1 -
 .../shape_check/tpcds_sf100/ddl/shape.tmpl    |   1 -
 .../tpcds_sf100/rf_prune/query1.groovy        |   1 -
 .../tpcds_sf100/rf_prune/query10.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query11.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query12.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query13.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query14.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query15.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query16.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query17.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query18.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query19.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query2.groovy        |   1 -
 .../tpcds_sf100/rf_prune/query20.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query21.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query22.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query24.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query25.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query26.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query27.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query28.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query29.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query3.groovy        |   1 -
 .../tpcds_sf100/rf_prune/query30.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query31.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query33.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query34.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query35.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query36.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query37.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query39.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query4.groovy        |   1 -
 .../tpcds_sf100/rf_prune/query40.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query41.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query42.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query43.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query44.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query45.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query46.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query47.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query48.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query49.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query5.groovy        |   1 -
 .../tpcds_sf100/rf_prune/query50.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query51.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query52.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query53.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query54.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query55.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query56.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query57.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query58.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query59.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query6.groovy        |   1 -
 .../tpcds_sf100/rf_prune/query60.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query61.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query62.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query63.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query64.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query65.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query66.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query67.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query68.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query69.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query7.groovy        |   1 -
 .../tpcds_sf100/rf_prune/query70.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query71.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query72.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query73.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query74.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query75.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query76.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query77.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query78.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query79.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query8.groovy        |   1 -
 .../tpcds_sf100/rf_prune/query80.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query81.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query82.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query83.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query84.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query85.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query86.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query87.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query88.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query89.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query9.groovy        |   1 -
 .../tpcds_sf100/rf_prune/query90.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query91.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query92.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query93.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query94.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query95.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query96.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query98.groovy       |   1 -
 .../tpcds_sf100/rf_prune/query99.groovy       |   1 -
 .../tpcds_sf100/shape/query1.groovy           |   1 -
 .../tpcds_sf100/shape/query10.groovy          |   1 -
 .../tpcds_sf100/shape/query11.groovy          |   1 -
 .../tpcds_sf100/shape/query12.groovy          |   1 -
 .../tpcds_sf100/shape/query13.groovy          |   1 -
 .../tpcds_sf100/shape/query14.groovy          |   1 -
 .../tpcds_sf100/shape/query15.groovy          |   1 -
 .../tpcds_sf100/shape/query16.groovy          |   1 -
 .../tpcds_sf100/shape/query17.groovy          |   1 -
 .../tpcds_sf100/shape/query18.groovy          |   1 -
 .../tpcds_sf100/shape/query19.groovy          |   1 -
 .../tpcds_sf100/shape/query2.groovy           |   1 -
 .../tpcds_sf100/shape/query20.groovy          |   1 -
 .../tpcds_sf100/shape/query21.groovy          |   1 -
 .../tpcds_sf100/shape/query22.groovy          |   1 -
 .../tpcds_sf100/shape/query24.groovy          |   1 -
 .../tpcds_sf100/shape/query25.groovy          |   1 -
 .../tpcds_sf100/shape/query26.groovy          |   1 -
 .../tpcds_sf100/shape/query27.groovy          |   1 -
 .../tpcds_sf100/shape/query28.groovy          |   1 -
 .../tpcds_sf100/shape/query29.groovy          |   1 -
 .../tpcds_sf100/shape/query3.groovy           |   1 -
 .../tpcds_sf100/shape/query30.groovy          |   1 -
 .../tpcds_sf100/shape/query31.groovy          |   1 -
 .../tpcds_sf100/shape/query33.groovy          |   1 -
 .../tpcds_sf100/shape/query34.groovy          |   1 -
 .../tpcds_sf100/shape/query35.groovy          |   1 -
 .../tpcds_sf100/shape/query36.groovy          |   1 -
 .../tpcds_sf100/shape/query37.groovy          |   1 -
 .../tpcds_sf100/shape/query39.groovy          |   1 -
 .../tpcds_sf100/shape/query4.groovy           |   1 -
 .../tpcds_sf100/shape/query40.groovy          |   1 -
 .../tpcds_sf100/shape/query41.groovy          |   1 -
 .../tpcds_sf100/shape/query42.groovy          |   1 -
 .../tpcds_sf100/shape/query43.groovy          |   1 -
 .../tpcds_sf100/shape/query44.groovy          |   1 -
 .../tpcds_sf100/shape/query45.groovy          |   1 -
 .../tpcds_sf100/shape/query46.groovy          |   1 -
 .../tpcds_sf100/shape/query47.groovy          |   1 -
 .../tpcds_sf100/shape/query48.groovy          |   1 -
 .../tpcds_sf100/shape/query49.groovy          |   1 -
 .../tpcds_sf100/shape/query5.groovy           |   1 -
 .../tpcds_sf100/shape/query50.groovy          |   1 -
 .../tpcds_sf100/shape/query51.groovy          |   1 -
 .../tpcds_sf100/shape/query52.groovy          |   1 -
 .../tpcds_sf100/shape/query53.groovy          |   1 -
 .../tpcds_sf100/shape/query54.groovy          |   1 -
 .../tpcds_sf100/shape/query55.groovy          |   1 -
 .../tpcds_sf100/shape/query56.groovy          |   1 -
 .../tpcds_sf100/shape/query57.groovy          |   1 -
 .../tpcds_sf100/shape/query58.groovy          |   1 -
 .../tpcds_sf100/shape/query59.groovy          |   1 -
 .../tpcds_sf100/shape/query6.groovy           |   1 -
 .../tpcds_sf100/shape/query60.groovy          |   1 -
 .../tpcds_sf100/shape/query61.groovy          |   1 -
 .../tpcds_sf100/shape/query62.groovy          |   1 -
 .../tpcds_sf100/shape/query63.groovy          |   1 -
 .../tpcds_sf100/shape/query64.groovy          |   1 -
 .../tpcds_sf100/shape/query65.groovy          |   1 -
 .../tpcds_sf100/shape/query66.groovy          |   1 -
 .../tpcds_sf100/shape/query67.groovy          |   1 -
 .../tpcds_sf100/shape/query68.groovy          |   1 -
 .../tpcds_sf100/shape/query69.groovy          |   1 -
 .../tpcds_sf100/shape/query7.groovy           |   1 -
 .../tpcds_sf100/shape/query70.groovy          |   1 -
 .../tpcds_sf100/shape/query71.groovy          |   1 -
 .../tpcds_sf100/shape/query72.groovy          |   1 -
 .../tpcds_sf100/shape/query73.groovy          |   1 -
 .../tpcds_sf100/shape/query74.groovy          |   1 -
 .../tpcds_sf100/shape/query75.groovy          |   1 -
 .../tpcds_sf100/shape/query76.groovy          |   1 -
 .../tpcds_sf100/shape/query77.groovy          |   1 -
 .../tpcds_sf100/shape/query78.groovy          |   1 -
 .../tpcds_sf100/shape/query79.groovy          |   1 -
 .../tpcds_sf100/shape/query8.groovy           |   1 -
 .../tpcds_sf100/shape/query80.groovy          |   1 -
 .../tpcds_sf100/shape/query81.groovy          |   1 -
 .../tpcds_sf100/shape/query82.groovy          |   1 -
 .../tpcds_sf100/shape/query83.groovy          |   1 -
 .../tpcds_sf100/shape/query84.groovy          |   1 -
 .../tpcds_sf100/shape/query85.groovy          |   1 -
 .../tpcds_sf100/shape/query86.groovy          |   1 -
 .../tpcds_sf100/shape/query87.groovy          |   1 -
 .../tpcds_sf100/shape/query88.groovy          |   1 -
 .../tpcds_sf100/shape/query89.groovy          |   1 -
 .../tpcds_sf100/shape/query9.groovy           |   1 -
 .../tpcds_sf100/shape/query90.groovy          |   1 -
 .../tpcds_sf100/shape/query91.groovy          |   1 -
 .../tpcds_sf100/shape/query92.groovy          |   1 -
 .../tpcds_sf100/shape/query93.groovy          |   1 -
 .../tpcds_sf100/shape/query94.groovy          |   1 -
 .../tpcds_sf100/shape/query95.groovy          |   1 -
 .../tpcds_sf100/shape/query96.groovy          |   1 -
 .../tpcds_sf100/shape/query98.groovy          |   1 -
 .../tpcds_sf100/shape/query99.groovy          |   1 -
 .../bs_downgrade_shape/query13.groovy         |   1 -
 .../bs_downgrade_shape/query19.groovy         |   1 -
 .../bs_downgrade_shape/query44.groovy         |   1 -
 .../bs_downgrade_shape/query45.groovy         |   1 -
 .../bs_downgrade_shape/query54.groovy         |   1 -
 .../bs_downgrade_shape/query56.groovy         |   1 -
 .../bs_downgrade_shape/query6.groovy          |   1 -
 .../bs_downgrade_shape/query61.groovy         |   1 -
 .../bs_downgrade_shape/query68.groovy         |   1 -
 .../bs_downgrade_shape/query8.groovy          |   1 -
 .../bs_downgrade_shape/query91.groovy         |   1 -
 .../bs_downgrade_shape/query95.groovy         |   1 -
 .../shape_check/tpcds_sf1000/ddl/shape.tmpl   |   1 -
 .../eliminate_empty/query10_empty.groovy      |   1 -
 .../tpcds_sf1000/hint/query1.groovy           |   1 -
 .../tpcds_sf1000/hint/query10.groovy          |   1 -
 .../tpcds_sf1000/hint/query11.groovy          |   1 -
 .../tpcds_sf1000/hint/query12.groovy          |   1 -
 .../tpcds_sf1000/hint/query13.groovy          |   1 -
 .../tpcds_sf1000/hint/query14.groovy          |   1 -
 .../tpcds_sf1000/hint/query15.groovy          |   1 -
 .../tpcds_sf1000/hint/query16.groovy          |   1 -
 .../tpcds_sf1000/hint/query17.groovy          |   1 -
 .../tpcds_sf1000/hint/query18.groovy          |   1 -
 .../tpcds_sf1000/hint/query19.groovy          |   1 -
 .../tpcds_sf1000/hint/query2.groovy           |   1 -
 .../tpcds_sf1000/hint/query20.groovy          |   1 -
 .../tpcds_sf1000/hint/query21.groovy          |   1 -
 .../tpcds_sf1000/hint/query22.groovy          |   1 -
 .../tpcds_sf1000/hint/query24.groovy          |   1 -
 .../tpcds_sf1000/hint/query25.groovy          |   1 -
 .../tpcds_sf1000/hint/query26.groovy          |   1 -
 .../tpcds_sf1000/hint/query27.groovy          |   1 -
 .../tpcds_sf1000/hint/query28.groovy          |   1 -
 .../tpcds_sf1000/hint/query29.groovy          |   1 -
 .../tpcds_sf1000/hint/query3.groovy           |   1 -
 .../tpcds_sf1000/hint/query30.groovy          |   1 -
 .../tpcds_sf1000/hint/query31.groovy          |   1 -
 .../tpcds_sf1000/hint/query34.groovy          |   1 -
 .../tpcds_sf1000/hint/query36.groovy          |   1 -
 .../tpcds_sf1000/hint/query37.groovy          |   1 -
 .../tpcds_sf1000/hint/query39.groovy          |   1 -
 .../tpcds_sf1000/hint/query4.groovy           |   1 -
 .../tpcds_sf1000/hint/query40.groovy          |   1 -
 .../tpcds_sf1000/hint/query41.groovy          |   1 -
 .../tpcds_sf1000/hint/query42.groovy          |   1 -
 .../tpcds_sf1000/hint/query43.groovy          |   1 -
 .../tpcds_sf1000/hint/query44.groovy          |   1 -
 .../tpcds_sf1000/hint/query45.groovy          |   1 -
 .../tpcds_sf1000/hint/query46.groovy          |   1 -
 .../tpcds_sf1000/hint/query47.groovy          |   1 -
 .../tpcds_sf1000/hint/query48.groovy          |   1 -
 .../tpcds_sf1000/hint/query49.groovy          |   1 -
 .../tpcds_sf1000/hint/query5.groovy           |   1 -
 .../tpcds_sf1000/hint/query50.groovy          |   1 -
 .../tpcds_sf1000/hint/query51.groovy          |   1 -
 .../tpcds_sf1000/hint/query52.groovy          |   1 -
 .../tpcds_sf1000/hint/query53.groovy          |   1 -
 .../tpcds_sf1000/hint/query54.groovy          |   1 -
 .../tpcds_sf1000/hint/query55.groovy          |   1 -
 .../tpcds_sf1000/hint/query56.groovy          |   1 -
 .../tpcds_sf1000/hint/query57.groovy          |   1 -
 .../tpcds_sf1000/hint/query58.groovy          |   1 -
 .../tpcds_sf1000/hint/query59.groovy          |   1 -
 .../tpcds_sf1000/hint/query6.groovy           |   1 -
 .../tpcds_sf1000/hint/query60.groovy          |   1 -
 .../tpcds_sf1000/hint/query61.groovy          |   1 -
 .../tpcds_sf1000/hint/query62.groovy          |   1 -
 .../tpcds_sf1000/hint/query63.groovy          |   1 -
 .../tpcds_sf1000/hint/query64.groovy          |   1 -
 .../tpcds_sf1000/hint/query65.groovy          |   1 -
 .../tpcds_sf1000/hint/query66.groovy          |   1 -
 .../tpcds_sf1000/hint/query67.groovy          |   1 -
 .../tpcds_sf1000/hint/query68.groovy          |   1 -
 .../tpcds_sf1000/hint/query69.groovy          |   1 -
 .../tpcds_sf1000/hint/query7.groovy           |   1 -
 .../tpcds_sf1000/hint/query70.groovy          |   1 -
 .../tpcds_sf1000/hint/query71.groovy          |   1 -
 .../tpcds_sf1000/hint/query72.groovy          |   1 -
 .../tpcds_sf1000/hint/query73.groovy          |   1 -
 .../tpcds_sf1000/hint/query74.groovy          |   1 -
 .../tpcds_sf1000/hint/query75.groovy          |   1 -
 .../tpcds_sf1000/hint/query76.groovy          |   1 -
 .../tpcds_sf1000/hint/query77.groovy          |   1 -
 .../tpcds_sf1000/hint/query78.groovy          |   1 -
 .../tpcds_sf1000/hint/query79.groovy          |   1 -
 .../tpcds_sf1000/hint/query8.groovy           |   1 -
 .../tpcds_sf1000/hint/query80.groovy          |   1 -
 .../tpcds_sf1000/hint/query81.groovy          |   1 -
 .../tpcds_sf1000/hint/query82.groovy          |   1 -
 .../tpcds_sf1000/hint/query84.groovy          |   1 -
 .../tpcds_sf1000/hint/query85.groovy          |   1 -
 .../tpcds_sf1000/hint/query86.groovy          |   1 -
 .../tpcds_sf1000/hint/query87.groovy          |   1 -
 .../tpcds_sf1000/hint/query88.groovy          |   1 -
 .../tpcds_sf1000/hint/query89.groovy          |   1 -
 .../tpcds_sf1000/hint/query9.groovy           |   1 -
 .../tpcds_sf1000/hint/query90.groovy          |   1 -
 .../tpcds_sf1000/hint/query91.groovy          |   1 -
 .../tpcds_sf1000/hint/query92.groovy          |   1 -
 .../tpcds_sf1000/hint/query93.groovy          |   1 -
 .../tpcds_sf1000/hint/query94.groovy          |   1 -
 .../tpcds_sf1000/hint/query95.groovy          |   1 -
 .../tpcds_sf1000/hint/query96.groovy          |   1 -
 .../tpcds_sf1000/hint/query98.groovy          |   1 -
 .../tpcds_sf1000/hint/query99.groovy          |   1 -
 .../tpcds_sf1000/shape/query1.groovy          |   1 -
 .../tpcds_sf1000/shape/query10.groovy         |   1 -
 .../tpcds_sf1000/shape/query11.groovy         |   1 -
 .../tpcds_sf1000/shape/query12.groovy         |   1 -
 .../tpcds_sf1000/shape/query13.groovy         |   1 -
 .../tpcds_sf1000/shape/query14.groovy         |   1 -
 .../tpcds_sf1000/shape/query15.groovy         |   1 -
 .../tpcds_sf1000/shape/query16.groovy         |   1 -
 .../tpcds_sf1000/shape/query17.groovy         |   1 -
 .../tpcds_sf1000/shape/query18.groovy         |   1 -
 .../tpcds_sf1000/shape/query19.groovy         |   1 -
 .../tpcds_sf1000/shape/query2.groovy          |   1 -
 .../tpcds_sf1000/shape/query20.groovy         |   1 -
 .../tpcds_sf1000/shape/query21.groovy         |   1 -
 .../tpcds_sf1000/shape/query22.groovy         |   1 -
 .../tpcds_sf1000/shape/query24.groovy         |   1 -
 .../tpcds_sf1000/shape/query25.groovy         |   1 -
 .../tpcds_sf1000/shape/query26.groovy         |   1 -
 .../tpcds_sf1000/shape/query27.groovy         |   1 -
 .../tpcds_sf1000/shape/query28.groovy         |   1 -
 .../tpcds_sf1000/shape/query29.groovy         |   1 -
 .../tpcds_sf1000/shape/query3.groovy          |   1 -
 .../tpcds_sf1000/shape/query30.groovy         |   1 -
 .../tpcds_sf1000/shape/query31.groovy         |   1 -
 .../tpcds_sf1000/shape/query33.groovy         |   1 -
 .../tpcds_sf1000/shape/query34.groovy         |   1 -
 .../tpcds_sf1000/shape/query35.groovy         |   1 -
 .../tpcds_sf1000/shape/query36.groovy         |   1 -
 .../tpcds_sf1000/shape/query37.groovy         |   1 -
 .../tpcds_sf1000/shape/query39.groovy         |   1 -
 .../tpcds_sf1000/shape/query4.groovy          |   1 -
 .../tpcds_sf1000/shape/query40.groovy         |   1 -
 .../tpcds_sf1000/shape/query41.groovy         |   1 -
 .../tpcds_sf1000/shape/query42.groovy         |   1 -
 .../tpcds_sf1000/shape/query43.groovy         |   1 -
 .../tpcds_sf1000/shape/query44.groovy         |   1 -
 .../tpcds_sf1000/shape/query45.groovy         |   1 -
 .../tpcds_sf1000/shape/query46.groovy         |   1 -
 .../tpcds_sf1000/shape/query47.groovy         |   1 -
 .../tpcds_sf1000/shape/query48.groovy         |   1 -
 .../tpcds_sf1000/shape/query49.groovy         |   1 -
 .../tpcds_sf1000/shape/query5.groovy          |   1 -
 .../tpcds_sf1000/shape/query50.groovy         |   1 -
 .../tpcds_sf1000/shape/query51.groovy         |   1 -
 .../tpcds_sf1000/shape/query52.groovy         |   1 -
 .../tpcds_sf1000/shape/query53.groovy         |   1 -
 .../tpcds_sf1000/shape/query54.groovy         |   1 -
 .../tpcds_sf1000/shape/query55.groovy         |   1 -
 .../tpcds_sf1000/shape/query56.groovy         |   1 -
 .../tpcds_sf1000/shape/query57.groovy         |   1 -
 .../tpcds_sf1000/shape/query58.groovy         |   1 -
 .../tpcds_sf1000/shape/query59.groovy         |   1 -
 .../tpcds_sf1000/shape/query6.groovy          |   1 -
 .../tpcds_sf1000/shape/query60.groovy         |   1 -
 .../tpcds_sf1000/shape/query61.groovy         |   1 -
 .../tpcds_sf1000/shape/query62.groovy         |   1 -
 .../tpcds_sf1000/shape/query63.groovy         |   1 -
 .../tpcds_sf1000/shape/query64.groovy         |   1 -
 .../tpcds_sf1000/shape/query65.groovy         |   1 -
 .../tpcds_sf1000/shape/query66.groovy         |   1 -
 .../tpcds_sf1000/shape/query67.groovy         |   1 -
 .../tpcds_sf1000/shape/query68.groovy         |   1 -
 .../tpcds_sf1000/shape/query69.groovy         |   1 -
 .../tpcds_sf1000/shape/query7.groovy          |   1 -
 .../tpcds_sf1000/shape/query70.groovy         |   1 -
 .../tpcds_sf1000/shape/query71.groovy         |   1 -
 .../tpcds_sf1000/shape/query72.groovy         |   1 -
 .../tpcds_sf1000/shape/query73.groovy         |   1 -
 .../tpcds_sf1000/shape/query74.groovy         |   1 -
 .../tpcds_sf1000/shape/query75.groovy         |   1 -
 .../tpcds_sf1000/shape/query76.groovy         |   1 -
 .../tpcds_sf1000/shape/query77.groovy         |   1 -
 .../tpcds_sf1000/shape/query78.groovy         |   1 -
 .../tpcds_sf1000/shape/query79.groovy         |   1 -
 .../tpcds_sf1000/shape/query8.groovy          |   1 -
 .../tpcds_sf1000/shape/query80.groovy         |   1 -
 .../tpcds_sf1000/shape/query81.groovy         |   1 -
 .../tpcds_sf1000/shape/query82.groovy         |   1 -
 .../tpcds_sf1000/shape/query83.groovy         |   1 -
 .../tpcds_sf1000/shape/query84.groovy         |   1 -
 .../tpcds_sf1000/shape/query85.groovy         |   1 -
 .../tpcds_sf1000/shape/query86.groovy         |   1 -
 .../tpcds_sf1000/shape/query87.groovy         |   1 -
 .../tpcds_sf1000/shape/query88.groovy         |   1 -
 .../tpcds_sf1000/shape/query89.groovy         |   1 -
 .../tpcds_sf1000/shape/query9.groovy          |   1 -
 .../tpcds_sf1000/shape/query90.groovy         |   1 -
 .../tpcds_sf1000/shape/query91.groovy         |   1 -
 .../tpcds_sf1000/shape/query92.groovy         |   1 -
 .../tpcds_sf1000/shape/query93.groovy         |   1 -
 .../tpcds_sf1000/shape/query94.groovy         |   1 -
 .../tpcds_sf1000/shape/query95.groovy         |   1 -
 .../tpcds_sf1000/shape/query96.groovy         |   1 -
 .../tpcds_sf1000/shape/query98.groovy         |   1 -
 .../tpcds_sf1000/shape/query99.groovy         |   1 -
 .../tpcds_sf10t_orc/ddl/shape.tmpl            |   1 -
 .../tpcds_sf10t_orc/shape/query1.groovy       |   2 +-
 .../tpcds_sf10t_orc/shape/query10.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query11.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query12.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query13.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query14.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query15.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query16.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query17.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query18.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query19.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query2.groovy       |   1 -
 .../tpcds_sf10t_orc/shape/query20.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query21.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query22.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query23.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query24.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query25.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query26.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query27.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query28.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query29.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query3.groovy       |   1 -
 .../tpcds_sf10t_orc/shape/query30.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query31.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query32.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query33.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query34.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query35.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query36.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query37.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query38.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query39.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query4.groovy       |   1 -
 .../tpcds_sf10t_orc/shape/query40.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query41.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query42.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query43.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query44.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query45.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query46.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query47.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query48.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query49.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query5.groovy       |   1 -
 .../tpcds_sf10t_orc/shape/query50.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query51.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query52.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query53.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query54.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query55.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query56.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query57.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query58.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query59.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query6.groovy       |   1 -
 .../tpcds_sf10t_orc/shape/query60.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query61.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query62.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query63.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query64.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query65.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query66.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query67.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query68.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query69.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query7.groovy       |   1 -
 .../tpcds_sf10t_orc/shape/query70.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query71.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query72.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query73.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query74.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query75.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query76.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query77.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query78.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query79.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query8.groovy       |   1 -
 .../tpcds_sf10t_orc/shape/query80.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query81.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query82.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query83.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query84.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query85.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query86.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query87.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query88.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query89.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query9.groovy       |   1 -
 .../tpcds_sf10t_orc/shape/query90.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query91.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query92.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query93.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query94.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query95.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query96.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query97.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query98.groovy      |   1 -
 .../tpcds_sf10t_orc/shape/query99.groovy      |   1 -
 .../runtime_filter/test_pushdown_setop.groovy |   1 -
 .../suites/ssb_sf100_p2/sql/bigbuild.sql      |   2 +-
 .../suites/ssb_sf1_p2/nereids/q1.1.groovy     |   2 +-
 .../suites/ssb_sf1_p2/nereids/q1.2.groovy     |   2 +-
 .../suites/ssb_sf1_p2/nereids/q1.3.groovy     |   2 +-
 .../suites/ssb_sf1_p2/nereids/q2.1.groovy     |   2 +-
 .../suites/ssb_sf1_p2/nereids/q2.2.groovy     |   2 +-
 .../suites/ssb_sf1_p2/nereids/q2.3.groovy     |   2 +-
 .../suites/ssb_sf1_p2/nereids/q3.1.groovy     |   2 +-
 .../suites/ssb_sf1_p2/nereids/q3.2.groovy     |   2 +-
 .../suites/ssb_sf1_p2/nereids/q3.3.groovy     |   2 +-
 .../suites/ssb_sf1_p2/nereids/q3.4.groovy     |   2 +-
 .../suites/ssb_sf1_p2/nereids/q4.1.groovy     |   2 +-
 .../suites/ssb_sf1_p2/nereids/q4.2.groovy     |   2 +-
 .../suites/ssb_sf1_p2/nereids/q4.3.groovy     |   2 +-
 .../functions_test/test_right_join.groovy     |   2 +-
 .../suites/tpch_sf1_p2/nereids/q1.groovy      |   2 +-
 .../suites/tpch_sf1_p2/nereids/q10.groovy     |   2 +-
 .../suites/tpch_sf1_p2/nereids/q11.groovy     |   2 +-
 .../suites/tpch_sf1_p2/nereids/q12.groovy     |   2 +-
 .../suites/tpch_sf1_p2/nereids/q13.groovy     |   2 +-
 .../suites/tpch_sf1_p2/nereids/q14.groovy     |   2 +-
 .../suites/tpch_sf1_p2/nereids/q15.groovy     |   2 +-
 .../suites/tpch_sf1_p2/nereids/q16.groovy     |   2 +-
 .../suites/tpch_sf1_p2/nereids/q17.groovy     |   2 +-
 .../suites/tpch_sf1_p2/nereids/q18.groovy     |   2 +-
 .../suites/tpch_sf1_p2/nereids/q19.groovy     |   2 +-
 .../suites/tpch_sf1_p2/nereids/q2.groovy      |   2 +-
 .../suites/tpch_sf1_p2/nereids/q20.groovy     |   2 +-
 .../suites/tpch_sf1_p2/nereids/q21.groovy     |   2 +-
 .../suites/tpch_sf1_p2/nereids/q22.groovy     |   2 +-
 .../suites/tpch_sf1_p2/nereids/q3.groovy      |   2 +-
 .../suites/tpch_sf1_p2/nereids/q4.groovy      |   2 +-
 .../suites/tpch_sf1_p2/nereids/q6.groovy      |   2 +-
 .../suites/tpch_sf1_p2/nereids/q7.groovy      |   2 +-
 .../suites/tpch_sf1_p2/nereids/q8.groovy      |   2 +-
 .../suites/tpch_sf1_p2/nereids/q9.groovy      |   2 +-
 regression-test/suites/variant_p0/load.groovy |   2 +-
 .../suites/variant_p0/nested.groovy           |   4 +-
 .../variant_p0/test_sub_path_pruning.groovy   |   2 +-
 .../workload_manager_p0/test_curd_wlg.groovy  |   4 +-
 609 files changed, 352 insertions(+), 857 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 5e663f0112699ac..0e20ed23f2c0759 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -977,8 +977,8 @@ public class SessionVariable implements Serializable, Writable {
      * the parallel exec instance num for one Fragment in one BE
      * 1 means disable this feature
      */
-    @VariableMgr.VarAttr(name = PARALLEL_FRAGMENT_EXEC_INSTANCE_NUM, needForward = true, fuzzy = true,
-                        setter = "setFragmentInstanceNum")
+    @VariableMgr.VarAttr(name = PARALLEL_FRAGMENT_EXEC_INSTANCE_NUM, needForward = true, fuzzy = false,
+                        setter = "setFragmentInstanceNum", varType = VariableAnnotation.DEPRECATED)
     public int parallelExecInstanceNum = 8;
 
     @VariableMgr.VarAttr(name = PARALLEL_PIPELINE_TASK_NUM, fuzzy = true, needForward = true,
@@ -2418,7 +2418,6 @@ public boolean isEnableSortSpill() {
     @SuppressWarnings("checkstyle:Indentation")
     public void initFuzzyModeVariables() {
         Random random = new SecureRandom();
-        this.parallelExecInstanceNum = random.nextInt(8) + 1;
         this.parallelPipelineTaskNum = random.nextInt(8);
         this.parallelPrepareThreshold = random.nextInt(32) + 1;
         this.enableCommonExprPushdown = random.nextBoolean();
@@ -2929,10 +2928,7 @@ public void setPipelineTaskNum(String value) throws Exception {
         this.parallelPipelineTaskNum = val;
     }
 
-    public void setFragmentInstanceNum(String value) throws Exception {
-        int val = checkFieldValue(PARALLEL_FRAGMENT_EXEC_INSTANCE_NUM, 1, value);
-        this.parallelExecInstanceNum = val;
-    }
+    public void setFragmentInstanceNum(String value) {}
 
     public void setOrcTinyStripeThresholdBytes(String value) throws Exception {
         long val = checkFieldLongValue(ORC_TINY_STRIPE_THRESHOLD_BYTES, 0, value);
diff --git a/pytest/qe/query_regression/result/issue_5171.result b/pytest/qe/query_regression/result/issue_5171.result
index d85a174e4cf9e22..2714df203fb7197 100644
--- a/pytest/qe/query_regression/result/issue_5171.result
+++ b/pytest/qe/query_regression/result/issue_5171.result
@@ -6,7 +6,7 @@ USE issue_5171
 CREATE TABLE `per` ( `A` bigint(20) NULL COMMENT "") ENGINE=OLAP DUPLICATE KEY(`A`) COMMENT "OLAP" DISTRIBUTED BY HASH(`A`) BUCKETS 5 PROPERTIES ( "replication_num" = "1", "in_memory" = "false", "storage_format" = "V2");
 ()
 insert into per values(65536);
-set parallel_fragment_exec_instance_num = 15;
+set PARALLEL_PIPELINE_TASK_NUM = 15;
 ()
 SELECT PERCENTILE_APPROX(A, 0.5) from per;
 (['65536.0'],)
diff --git a/pytest/qe/query_regression/result/issue_8778.result b/pytest/qe/query_regression/result/issue_8778.result
index 8a5cfe42f89b263..738a8cfc345ecaf 100644
--- a/pytest/qe/query_regression/result/issue_8778.result
+++ b/pytest/qe/query_regression/result/issue_8778.result
@@ -5,7 +5,7 @@ CREATE DATABASE issue_8778;
 use issue_8778
 CREATE TABLE `t1` (`tc1` int(11) NULL COMMENT "", `tc2` int(11) NULL COMMENT "", `tc3` int(11) NULL COMMENT "") ENGINE=OLAP DUPLICATE KEY(`tc1`, `tc2`, `tc3`) COMMENT "OLAP" DISTRIBUTED BY HASH(`tc1`) BUCKETS 10 PROPERTIES ("replication_allocation" = "tag.location.default: 1", "in_memory" = "false", "storage_format" = "V2");
 ()
-set parallel_fragment_exec_instance_num=2;
+set PARALLEL_PIPELINE_TASK_NUM=2;
 ()
 set disable_colocate_plan=false;
 ()
diff --git a/pytest/qe/query_regression/sql/issue_5171.sql b/pytest/qe/query_regression/sql/issue_5171.sql
index e72decd24470dd7..769ab0bba3d2bb2 100644
--- a/pytest/qe/query_regression/sql/issue_5171.sql
+++ b/pytest/qe/query_regression/sql/issue_5171.sql
@@ -7,7 +7,7 @@ CREATE TABLE `per` ( `A` bigint(20) NULL COMMENT "") ENGINE=OLAP DUPLICATE KEY(`
 
 insert into per values(65536);
 
-set parallel_fragment_exec_instance_num = 15;
+set PARALLEL_PIPELINE_TASK_NUM = 15;
 
 SELECT PERCENTILE_APPROX(A, 0.5) from per;
 
diff --git a/pytest/qe/query_regression/sql/issue_8778.sql b/pytest/qe/query_regression/sql/issue_8778.sql
index 8df59aed1fc7aa2..8990bec8d013464 100644
--- a/pytest/qe/query_regression/sql/issue_8778.sql
+++ b/pytest/qe/query_regression/sql/issue_8778.sql
@@ -3,7 +3,7 @@ DROP DATABASE IF EXISTS issue_8778;
 CREATE DATABASE issue_8778;
 use issue_8778
 CREATE TABLE `t1` (`tc1` int(11) NULL COMMENT "", `tc2` int(11) NULL COMMENT "", `tc3` int(11) NULL COMMENT "") ENGINE=OLAP DUPLICATE KEY(`tc1`, `tc2`, `tc3`) COMMENT "OLAP" DISTRIBUTED BY HASH(`tc1`) BUCKETS 10 PROPERTIES ("replication_allocation" = "tag.location.default: 1", "in_memory" = "false", "storage_format" = "V2");
-set parallel_fragment_exec_instance_num=2;
+set PARALLEL_PIPELINE_TASK_NUM=2;
 set disable_colocate_plan=false;
 insert into t1 values(1,2,1),(1,3,1),(2,1,1),(3,1,1);
 select t1.tc1,t1.tc2,sum(t1.tc3) as total from t1 join[shuffle] t1 t2 on t1.tc1=t2.tc1 group by rollup(tc1,tc2) order by t1.tc1,t1.tc2,total;
diff --git a/regression-test/pipeline/performance/clickbench/conf/opt_session_variables.sql b/regression-test/pipeline/performance/clickbench/conf/opt_session_variables.sql
index 24516a734e34b29..742d613c14f44c3 100644
--- a/regression-test/pipeline/performance/clickbench/conf/opt_session_variables.sql
+++ b/regression-test/pipeline/performance/clickbench/conf/opt_session_variables.sql
@@ -1,5 +1,5 @@
 set global exec_mem_limit=34359738368;
-set global parallel_fragment_exec_instance_num=16;
+set global parallel_pipeline_task_num=16;
 set global parallel_pipeline_task_num=16;
 set global enable_single_distinct_column_opt=true;
 set global enable_function_pushdown=true;
diff --git a/regression-test/suites/correctness_p0/test_null_aware_left_anti_join.groovy b/regression-test/suites/correctness_p0/test_null_aware_left_anti_join.groovy
index 9260ff81ab6a72e..1898be830b2100f 100644
--- a/regression-test/suites/correctness_p0/test_null_aware_left_anti_join.groovy
+++ b/regression-test/suites/correctness_p0/test_null_aware_left_anti_join.groovy
@@ -56,7 +56,6 @@ suite("test_null_aware_left_anti_join") {
 
     qt_select """ select ${tableName2}.k1 from ${tableName2} where k1 not in (select ${tableName1}.k1 from ${tableName1}) order by ${tableName2}.k1; """
 
-    sql """ set parallel_fragment_exec_instance_num=2; """
     sql """ set parallel_pipeline_task_num=2; """
     qt_select """ select ${tableName2}.k1 from ${tableName2} where k1 not in (select ${tableName1}.k1 from ${tableName1}) order by ${tableName2}.k1; """
 
diff --git a/regression-test/suites/correctness_p0/test_scan_topn_limit.groovy b/regression-test/suites/correctness_p0/test_scan_topn_limit.groovy
index cc819d20667b764..de1f4393bad1bf0 100644
--- a/regression-test/suites/correctness_p0/test_scan_topn_limit.groovy
+++ b/regression-test/suites/correctness_p0/test_scan_topn_limit.groovy
@@ -35,6 +35,6 @@ suite("test_scan_topn_limit") {
         sql """ INSERT INTO ${tableName} values (${i}, 100-${i}) """
     }
     qt_select """ SELECT /*+ SET_VAR(experimental_enable_pipeline_engine = true,
-                            parallel_fragment_exec_instance_num = 4) */ 
+                            parallel_pipeline_task_num = 4) */
                         k1, k2 FROM ${tableName} ORDER BY k1 DESC, k2 LIMIT 10 """
 }
diff --git a/regression-test/suites/datatype_p0/nested_types/query/test_nested_type_with_resize.groovy b/regression-test/suites/datatype_p0/nested_types/query/test_nested_type_with_resize.groovy
index c68107dcd37a917..c02f0908ed669aa 100644
--- a/regression-test/suites/datatype_p0/nested_types/query/test_nested_type_with_resize.groovy
+++ b/regression-test/suites/datatype_p0/nested_types/query/test_nested_type_with_resize.groovy
@@ -65,9 +65,9 @@ suite("test_nested_type_with_resize") {
         }
     }
 
-    order_qt_sql """ /*set ShuffleSendBytes=0|ShuffleSendRows=0|FuzzyVariables=batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=3,parallel_pipeline_task_num=5,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=true,parallel_scan_max_scanners_count=48,parallel_scan_min_rows_per_scanner=16384,parallel_prepare_threshold=13,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=12,enable_parallel_result_sink=true,sort_phase_num=0,rewrite_or_to_in_predicate_threshold=100000,enable_function_pushdown=false,enable_common_expr_pushdown=true,enable_local_exchange=false,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_delete_sub_predicate_v2=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5*/ select col4 from test_array_resize order by col1,col2,col3 limit 10; """
-    order_qt_sql """ /*set ShuffleSendBytes=0|ShuffleSendRows=0|FuzzyVariables=batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=3,parallel_pipeline_task_num=5,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=true,parallel_scan_max_scanners_count=48,parallel_scan_min_rows_per_scanner=16384,parallel_prepare_threshold=13,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=12,enable_parallel_result_sink=true,sort_phase_num=0,rewrite_or_to_in_predicate_threshold=100000,enable_function_pushdown=false,enable_common_expr_pushdown=true,enable_local_exchange=false,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_delete_sub_predicate_v2=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5*/ select col22 from test_array_resize order by col1,col2,col3 limit 10; """
-    order_qt_sql """ /*set ShuffleSendBytes=0|ShuffleSendRows=0|FuzzyVariables=batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=3,parallel_pipeline_task_num=5,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=true,parallel_scan_max_scanners_count=48,parallel_scan_min_rows_per_scanner=16384,parallel_prepare_threshold=13,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=12,enable_parallel_result_sink=true,sort_phase_num=0,rewrite_or_to_in_predicate_threshold=100000,enable_function_pushdown=false,enable_common_expr_pushdown=true,enable_local_exchange=false,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_delete_sub_predicate_v2=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5*/ select col32 from test_array_resize order by col1,col2,col3 limit 10; """
-    order_qt_sql """ /*set ShuffleSendBytes=0|ShuffleSendRows=0|FuzzyVariables=batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=3,parallel_pipeline_task_num=5,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=true,parallel_scan_max_scanners_count=48,parallel_scan_min_rows_per_scanner=16384,parallel_prepare_threshold=13,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=12,enable_parallel_result_sink=true,sort_phase_num=0,rewrite_or_to_in_predicate_threshold=100000,enable_function_pushdown=false,enable_common_expr_pushdown=true,enable_local_exchange=false,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_delete_sub_predicate_v2=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5*/ select col74 from test_array_resize order by col1,col2,col3 limit 10; """
+    order_qt_sql """ /*set ShuffleSendBytes=0|ShuffleSendRows=0|FuzzyVariables=batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_pipeline_task_num=5,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=true,parallel_scan_max_scanners_count=48,parallel_scan_min_rows_per_scanner=16384,parallel_prepare_threshold=13,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=12,enable_parallel_result_sink=true,sort_phase_num=0,rewrite_or_to_in_predicate_threshold=100000,enable_function_pushdown=false,enable_common_expr_pushdown=true,enable_local_exchange=false,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_delete_sub_predicate_v2=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5*/ select col4 from test_array_resize order by col1,col2,col3 limit 10; """
+    order_qt_sql """ /*set ShuffleSendBytes=0|ShuffleSendRows=0|FuzzyVariables=batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_pipeline_task_num=5,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=true,parallel_scan_max_scanners_count=48,parallel_scan_min_rows_per_scanner=16384,parallel_prepare_threshold=13,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=12,enable_parallel_result_sink=true,sort_phase_num=0,rewrite_or_to_in_predicate_threshold=100000,enable_function_pushdown=false,enable_common_expr_pushdown=true,enable_local_exchange=false,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_delete_sub_predicate_v2=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5*/ select col22 from test_array_resize order by col1,col2,col3 limit 10; """
+    order_qt_sql """ /*set ShuffleSendBytes=0|ShuffleSendRows=0|FuzzyVariables=batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_pipeline_task_num=5,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=true,parallel_scan_max_scanners_count=48,parallel_scan_min_rows_per_scanner=16384,parallel_prepare_threshold=13,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=12,enable_parallel_result_sink=true,sort_phase_num=0,rewrite_or_to_in_predicate_threshold=100000,enable_function_pushdown=false,enable_common_expr_pushdown=true,enable_local_exchange=false,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_delete_sub_predicate_v2=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5*/ select col32 from test_array_resize order by col1,col2,col3 limit 10; """
+    order_qt_sql """ /*set ShuffleSendBytes=0|ShuffleSendRows=0|FuzzyVariables=batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_pipeline_task_num=5,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=true,parallel_scan_max_scanners_count=48,parallel_scan_min_rows_per_scanner=16384,parallel_prepare_threshold=13,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=12,enable_parallel_result_sink=true,sort_phase_num=0,rewrite_or_to_in_predicate_threshold=100000,enable_function_pushdown=false,enable_common_expr_pushdown=true,enable_local_exchange=false,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_delete_sub_predicate_v2=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5*/ select col74 from test_array_resize order by col1,col2,col3 limit 10; """
 
 }
diff --git a/regression-test/suites/external_table_p0/hive/hive_tpch_sf1_orc.groovy b/regression-test/suites/external_table_p0/hive/hive_tpch_sf1_orc.groovy
index be8007c3016285b..9834ab5550ab620 100644
--- a/regression-test/suites/external_table_p0/hive/hive_tpch_sf1_orc.groovy
+++ b/regression-test/suites/external_table_p0/hive/hive_tpch_sf1_orc.groovy
@@ -22,7 +22,7 @@ suite("test_catalog_hive_orc", "p0,external,hive,external_docker,external_docker
     def q01 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q01 """
@@ -53,7 +53,7 @@ order by
     def q02 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=2"""
+        sql """set parallel_pipeline_task_num=2"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         sql """set enable_projection=true"""
@@ -103,7 +103,7 @@ limit 100;
     def q03 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         sql """set enable_projection=true"""
@@ -137,7 +137,7 @@ limit 10;
     def q04 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=1"""
+        sql """set parallel_pipeline_task_num=1"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q04 """
@@ -167,7 +167,7 @@ order by
     def q05 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q05 """
@@ -189,7 +189,7 @@ order by revenue desc;
     def q06 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=1"""
+        sql """set parallel_pipeline_task_num=1"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q06 """
@@ -208,7 +208,7 @@ where
     def q07 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=4"""
+        sql """set parallel_pipeline_task_num=4"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q07 """
@@ -249,7 +249,7 @@ order by
     def q08 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=1"""
+        sql """set parallel_pipeline_task_num=1"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q08 """
@@ -296,7 +296,7 @@ order by
     def q09 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=4"""
+        sql """set parallel_pipeline_task_num=4"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q09 """
@@ -329,7 +329,7 @@ order by
     def q10 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q10 """
@@ -372,7 +372,7 @@ limit 20;
     def q11 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=2"""
+        sql """set parallel_pipeline_task_num=2"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q11 """
@@ -409,7 +409,7 @@ order by
     def q12 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=2"""
+        sql """set parallel_pipeline_task_num=2"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=true"""
         qt_q12 """
@@ -446,7 +446,7 @@ order by
     def q13 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=4"""
+        sql """set parallel_pipeline_task_num=4"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q13 """
@@ -476,7 +476,7 @@ order by
     def q14 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q14 """
@@ -499,7 +499,7 @@ where
     def q15 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=true"""
         qt_q15 """
@@ -539,7 +539,7 @@ order by
     def q16 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=true"""
         qt_q16 """
@@ -579,7 +579,7 @@ order by
     def q17 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=1"""
+        sql """set parallel_pipeline_task_num=1"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=true"""
         qt_q17 """
@@ -608,7 +608,7 @@ where
     def q18 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q18 """
@@ -656,7 +656,7 @@ limit 100;
     def q19 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q19 """
@@ -701,7 +701,7 @@ where
     def q20 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=true"""
         qt_q20 """
@@ -736,7 +736,7 @@ order by s_name;
     def q21 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=true"""
         qt_q21 """
@@ -777,7 +777,7 @@ limit 100;
     def q22 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=true"""
         qt_q22 """
diff --git a/regression-test/suites/external_table_p0/hive/hive_tpch_sf1_parquet.groovy b/regression-test/suites/external_table_p0/hive/hive_tpch_sf1_parquet.groovy
index 6750e57319ff00e..d167a51353d9caf 100644
--- a/regression-test/suites/external_table_p0/hive/hive_tpch_sf1_parquet.groovy
+++ b/regression-test/suites/external_table_p0/hive/hive_tpch_sf1_parquet.groovy
@@ -22,7 +22,7 @@ suite("test_catalog_hive_parquet", "p0,external,hive,external_docker,external_do
     def q01 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q01 """
@@ -53,7 +53,7 @@ order by
     def q02 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=2"""
+        sql """set parallel_pipeline_task_num=2"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         sql """set enable_projection=true"""
@@ -103,7 +103,7 @@ limit 100;
     def q03 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         sql """set enable_projection=true"""
@@ -137,7 +137,7 @@ limit 10;
     def q04 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=1"""
+        sql """set parallel_pipeline_task_num=1"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q04 """
@@ -167,7 +167,7 @@ order by
     def q05 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q05 """
@@ -189,7 +189,7 @@ order by revenue desc;
     def q06 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=1"""
+        sql """set parallel_pipeline_task_num=1"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q06 """
@@ -208,7 +208,7 @@ where
     def q07 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=4"""
+        sql """set parallel_pipeline_task_num=4"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q07 """
@@ -249,7 +249,7 @@ order by
     def q08 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=1"""
+        sql """set parallel_pipeline_task_num=1"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q08 """
@@ -296,7 +296,7 @@ order by
     def q09 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=4"""
+        sql """set parallel_pipeline_task_num=4"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q09 """
@@ -329,7 +329,7 @@ order by
     def q10 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q10 """
@@ -372,7 +372,7 @@ limit 20;
     def q11 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=2"""
+        sql """set parallel_pipeline_task_num=2"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q11 """
@@ -409,7 +409,7 @@ order by
     def q12 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=2"""
+        sql """set parallel_pipeline_task_num=2"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=true"""
         qt_q12 """
@@ -446,7 +446,7 @@ order by
     def q13 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=4"""
+        sql """set parallel_pipeline_task_num=4"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q13 """
@@ -476,7 +476,7 @@ order by
     def q14 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q14 """
@@ -499,7 +499,7 @@ where
     def q15 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=true"""
         qt_q15 """
@@ -539,7 +539,7 @@ order by
     def q16 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=true"""
         qt_q16 """
@@ -579,7 +579,7 @@ order by
     def q17 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=1"""
+        sql """set parallel_pipeline_task_num=1"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=true"""
         qt_q17 """
@@ -608,7 +608,7 @@ where
     def q18 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q18 """
@@ -656,7 +656,7 @@ limit 100;
     def q19 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q19 """
@@ -701,7 +701,7 @@ where
     def q20 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=true"""
         qt_q20 """
@@ -736,7 +736,7 @@ order by s_name;
     def q21 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=true"""
         qt_q21 """
@@ -777,7 +777,7 @@ limit 100;
     def q22 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=true"""
         qt_q22 """
diff --git a/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_tpch_sf1_orc.groovy b/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_tpch_sf1_orc.groovy
index 6ee38e0021d4506..d41f97e1df295f6 100644
--- a/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_tpch_sf1_orc.groovy
+++ b/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_tpch_sf1_orc.groovy
@@ -21,7 +21,7 @@ suite("test_trino_hive_tpch_sf1_orc", "p0,external,hive,external_docker,external
     def q01 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q01 """
@@ -52,7 +52,7 @@ order by
     def q02 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=2"""
+        sql """set parallel_pipeline_task_num=2"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         sql """set enable_projection=true"""
@@ -102,7 +102,7 @@ limit 100;
     def q03 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         sql """set enable_projection=true"""
@@ -136,7 +136,7 @@ limit 10;
     def q04 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=1"""
+        sql """set parallel_pipeline_task_num=1"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q04 """
@@ -166,7 +166,7 @@ order by
     def q05 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q05 """
@@ -188,7 +188,7 @@ order by revenue desc;
     def q06 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=1"""
+        sql """set parallel_pipeline_task_num=1"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q06 """
@@ -207,7 +207,7 @@ where
     def q07 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=4"""
+        sql """set parallel_pipeline_task_num=4"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q07 """
@@ -248,7 +248,7 @@ order by
     def q08 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=1"""
+        sql """set parallel_pipeline_task_num=1"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q08 """
@@ -295,7 +295,7 @@ order by
     def q09 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=4"""
+        sql """set parallel_pipeline_task_num=4"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q09 """
@@ -328,7 +328,7 @@ order by
     def q10 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q10 """
@@ -371,7 +371,7 @@ limit 20;
     def q11 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=2"""
+        sql """set parallel_pipeline_task_num=2"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q11 """
@@ -408,7 +408,7 @@ order by
     def q12 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=2"""
+        sql """set parallel_pipeline_task_num=2"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=true"""
         qt_q12 """
@@ -445,7 +445,7 @@ order by
     def q13 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=4"""
+        sql """set parallel_pipeline_task_num=4"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q13 """
@@ -475,7 +475,7 @@ order by
     def q14 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q14 """
@@ -498,7 +498,7 @@ where
     def q15 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=true"""
         qt_q15 """
@@ -538,7 +538,7 @@ order by
     def q16 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=true"""
         qt_q16 """
@@ -578,7 +578,7 @@ order by
     def q17 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=1"""
+        sql """set parallel_pipeline_task_num=1"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=true"""
         qt_q17 """
@@ -607,7 +607,7 @@ where
     def q18 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q18 """
@@ -655,7 +655,7 @@ limit 100;
     def q19 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q19 """
@@ -700,7 +700,7 @@ where
     def q20 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=true"""
         qt_q20 """
@@ -735,7 +735,7 @@ order by s_name;
     def q21 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=true"""
         qt_q21 """
@@ -776,7 +776,7 @@ limit 100;
     def q22 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=true"""
         qt_q22 """
diff --git a/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_tpch_sf1_parquet.groovy b/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_tpch_sf1_parquet.groovy
index 8bd86c7def161a6..ae30fac410b3faa 100644
--- a/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_tpch_sf1_parquet.groovy
+++ b/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_tpch_sf1_parquet.groovy
@@ -21,7 +21,7 @@ suite("test_trino_hive_tpch_sf1_parquet", "p0,external,hive,external_docker,exte
     def q01 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q01 """
@@ -52,7 +52,7 @@ order by
     def q02 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=2"""
+        sql """set parallel_pipeline_task_num=2"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         sql """set enable_projection=true"""
@@ -102,7 +102,7 @@ limit 100;
     def q03 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         sql """set enable_projection=true"""
@@ -136,7 +136,7 @@ limit 10;
     def q04 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=1"""
+        sql """set parallel_pipeline_task_num=1"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q04 """
@@ -166,7 +166,7 @@ order by
     def q05 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q05 """
@@ -188,7 +188,7 @@ order by revenue desc;
     def q06 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=1"""
+        sql """set parallel_pipeline_task_num=1"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q06 """
@@ -207,7 +207,7 @@ where
     def q07 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=4"""
+        sql """set parallel_pipeline_task_num=4"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q07 """
@@ -248,7 +248,7 @@ order by
     def q08 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=1"""
+        sql """set parallel_pipeline_task_num=1"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q08 """
@@ -295,7 +295,7 @@ order by
     def q09 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=4"""
+        sql """set parallel_pipeline_task_num=4"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q09 """
@@ -328,7 +328,7 @@ order by
     def q10 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q10 """
@@ -371,7 +371,7 @@ limit 20;
     def q11 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=2"""
+        sql """set parallel_pipeline_task_num=2"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q11 """
@@ -408,7 +408,7 @@ order by
     def q12 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=2"""
+        sql """set parallel_pipeline_task_num=2"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=true"""
         qt_q12 """
@@ -445,7 +445,7 @@ order by
     def q13 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=4"""
+        sql """set parallel_pipeline_task_num=4"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q13 """
@@ -475,7 +475,7 @@ order by
     def q14 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q14 """
@@ -498,7 +498,7 @@ where
     def q15 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=true"""
         qt_q15 """
@@ -538,7 +538,7 @@ order by
     def q16 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=true"""
         qt_q16 """
@@ -578,7 +578,7 @@ order by
     def q17 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=1"""
+        sql """set parallel_pipeline_task_num=1"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=true"""
         qt_q17 """
@@ -607,7 +607,7 @@ where
     def q18 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q18 """
@@ -655,7 +655,7 @@ limit 100;
     def q19 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=false"""
         qt_q19 """
@@ -700,7 +700,7 @@ where
     def q20 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=true"""
         qt_q20 """
@@ -735,7 +735,7 @@ order by s_name;
     def q21 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=true"""
         sql """set enable_cost_based_join_reorder=true"""
         qt_q21 """
@@ -776,7 +776,7 @@ limit 100;
     def q22 = { 
         sql """set exec_mem_limit=8589934592"""
         sql """set enable_file_cache=${enable_file_cache}"""
-        sql """set parallel_fragment_exec_instance_num=8"""
+        sql """set parallel_pipeline_task_num=8"""
         sql """set disable_join_reorder=false"""
         sql """set enable_cost_based_join_reorder=true"""
         qt_q22 """
diff --git a/regression-test/suites/external_table_p2/hive/test_external_github.groovy b/regression-test/suites/external_table_p2/hive/test_external_github.groovy
index 2d456b9845543f8..431f4f9f76525d7 100644
--- a/regression-test/suites/external_table_p2/hive/test_external_github.groovy
+++ b/regression-test/suites/external_table_p2/hive/test_external_github.groovy
@@ -274,8 +274,8 @@ suite("test_external_github", "p2,external,hive,external_remote,external_remote_
         GROUP BY repo_name
         ORDER BY count() DESC
         LIMIT 50"""
-    def repositoriesWithClickhouse_related_comments1 = """SELECT /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=8, parallel_pipeline_task_num=8, query_timeout=600) */repo_name, count() FROM github_eventsSUFFIX WHERE lower(body) LIKE '%clickhouse%' GROUP BY repo_name ORDER BY count() DESC, repo_name ASC LIMIT 50"""
-    def repositoriesWithClickhouse_related_comments2 = """SELECT /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=8, parallel_pipeline_task_num=8, query_timeout=600) */
+    def repositoriesWithClickhouse_related_comments1 = """SELECT /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=8, parallel_pipeline_task_num=8, query_timeout=600) */repo_name, count() FROM github_eventsSUFFIX WHERE lower(body) LIKE '%clickhouse%' GROUP BY repo_name ORDER BY count() DESC, repo_name ASC LIMIT 50"""
+    def repositoriesWithClickhouse_related_comments2 = """SELECT /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=8, parallel_pipeline_task_num=8, query_timeout=600) */
             repo_name,
             sum(num_star) AS num_stars,
             sum(num_comment) AS num_comments
@@ -292,8 +292,8 @@ suite("test_external_github", "p2,external,hive,external_remote,external_remote_
         HAVING num_comments > 0
         ORDER BY num_stars DESC,num_comments DESC,repo_name ASC
         LIMIT 50"""
-    def repositoriesWithDoris_related_comments1 = """SELECT /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=8, parallel_fragment_exec_instance_num=8, query_timeout=600) */repo_name, count() FROM github_eventsSUFFIX WHERE lower(body) LIKE '%doris%' GROUP BY repo_name ORDER BY count() DESC, repo_name ASC LIMIT 50"""
-    def repositoriesWithDoris_related_comments2 = """SELECT /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=8, parallel_fragment_exec_instance_num=8, query_timeout=600) */
+    def repositoriesWithDoris_related_comments1 = """SELECT /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=8,query_timeout=600) */repo_name, count() FROM github_eventsSUFFIX WHERE lower(body) LIKE '%doris%' GROUP BY repo_name ORDER BY count() DESC, repo_name ASC LIMIT 50"""
+    def repositoriesWithDoris_related_comments2 = """SELECT /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=8,query_timeout=600) */
             repo_name,
             sum(num_star) AS num_stars,
             sum(num_comment) AS num_comments
diff --git a/regression-test/suites/nereids_hint_tpcds_p0/ddl/shape.tmpl b/regression-test/suites/nereids_hint_tpcds_p0/ddl/shape.tmpl
index d3556b96a345914..0f3462c785908cc 100644
--- a/regression-test/suites/nereids_hint_tpcds_p0/ddl/shape.tmpl
+++ b/regression-test/suites/nereids_hint_tpcds_p0/ddl/shape.tmpl
@@ -25,7 +25,7 @@ suite("query{--}") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
+    sql 'set parallel_pipeline_task_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/nereids_p0/cte/test_cte_filter_pushdown.groovy b/regression-test/suites/nereids_p0/cte/test_cte_filter_pushdown.groovy
index 795d1ae7e5cf40d..528dde0752dbcd1 100644
--- a/regression-test/suites/nereids_p0/cte/test_cte_filter_pushdown.groovy
+++ b/regression-test/suites/nereids_p0/cte/test_cte_filter_pushdown.groovy
@@ -47,7 +47,7 @@ suite("test_cte_filter_pushdown") {
     """
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
+    sql 'set parallel_pipeline_task_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/nereids_p0/join/test_join.groovy b/regression-test/suites/nereids_p0/join/test_join.groovy
index 5336c33913b2ae7..bc336429ea51e55 100644
--- a/regression-test/suites/nereids_p0/join/test_join.groovy
+++ b/regression-test/suites/nereids_p0/join/test_join.groovy
@@ -18,7 +18,7 @@
 suite("test_join", "nereids_p0") {
     sql "SET enable_nereids_planner=true"
     sql "SET enable_fallback_to_original_planner=false"
-    sql 'set parallel_fragment_exec_instance_num = 2;'
+    sql 'set parallel_pipeline_task_num = 2;'
     sql"use nereids_test_query_db"
 
     def tbName1 = "test"
diff --git a/regression-test/suites/nereids_p0/join/test_join5.groovy b/regression-test/suites/nereids_p0/join/test_join5.groovy
index 12844c275f7893e..91fed367006b15c 100644
--- a/regression-test/suites/nereids_p0/join/test_join5.groovy
+++ b/regression-test/suites/nereids_p0/join/test_join5.groovy
@@ -119,7 +119,7 @@ suite("test_join5", "nereids_p0") {
     sql " insert into c (name, a) values ('B', 'q');"
     sql " insert into c (name, a) values ('C', null);"
     
-    sql """set parallel_fragment_exec_instance_num=8"""
+    sql """set parallel_pipeline_task_num=8"""
         
     qt_join5 """
         select c.name, ss.code, ss.b_cnt, ss.const
diff --git a/regression-test/suites/nereids_p0/join/test_join_11.groovy b/regression-test/suites/nereids_p0/join/test_join_11.groovy
index 4fd2702a9ccb42d..82ffd4ffa95b584 100644
--- a/regression-test/suites/nereids_p0/join/test_join_11.groovy
+++ b/regression-test/suites/nereids_p0/join/test_join_11.groovy
@@ -18,7 +18,7 @@
 suite("test_join_11", "nereids_p0") {
     sql "SET enable_nereids_planner=true"
     sql "SET enable_fallback_to_original_planner=false"
-    sql 'set parallel_fragment_exec_instance_num = 2;'
+    sql 'set parallel_pipeline_task_num = 2;'
     sql "use nereids_test_query_db"
 
     def tbName1 = "test"
diff --git a/regression-test/suites/nereids_p0/join/test_join_12.groovy b/regression-test/suites/nereids_p0/join/test_join_12.groovy
index d1cf1de99fb0dc2..0e156d4752cbfb1 100644
--- a/regression-test/suites/nereids_p0/join/test_join_12.groovy
+++ b/regression-test/suites/nereids_p0/join/test_join_12.groovy
@@ -18,7 +18,7 @@
 suite("test_join_12", "nereids_p0") {
     sql "SET enable_nereids_planner=true"
     sql "SET enable_fallback_to_original_planner=false"
-    sql 'set parallel_fragment_exec_instance_num = 2;'
+    sql 'set parallel_pipeline_task_num = 2;'
     sql "use nereids_test_query_db"
 
     def tbName1 = "test"
diff --git a/regression-test/suites/nereids_p0/join/test_join_13.groovy b/regression-test/suites/nereids_p0/join/test_join_13.groovy
index e26a6cc345464e0..f9c8036cc60b0f2 100644
--- a/regression-test/suites/nereids_p0/join/test_join_13.groovy
+++ b/regression-test/suites/nereids_p0/join/test_join_13.groovy
@@ -18,7 +18,7 @@
 suite("test_join_13", "nereids_p0") {
     sql "SET enable_nereids_planner=true"
     sql "SET enable_fallback_to_original_planner=false"
-    sql 'set parallel_fragment_exec_instance_num = 2;'
+    sql 'set parallel_pipeline_task_num = 2;'
     sql "use nereids_test_query_db"
 
     def tbName1 = "test"
diff --git a/regression-test/suites/nereids_p0/join/test_join_14.groovy b/regression-test/suites/nereids_p0/join/test_join_14.groovy
index 95a8d954d357ede..364bb0eb62fd0ed 100644
--- a/regression-test/suites/nereids_p0/join/test_join_14.groovy
+++ b/regression-test/suites/nereids_p0/join/test_join_14.groovy
@@ -18,7 +18,7 @@
 suite("test_join_14", "nereids_p0") {
     sql "SET enable_nereids_planner=true"
     sql "SET enable_fallback_to_original_planner=false"
-    sql 'set parallel_fragment_exec_instance_num = 2;'
+    sql 'set parallel_pipeline_task_num = 2;'
     sql "use nereids_test_query_db"
 
     def tbName1 = "test"
diff --git a/regression-test/suites/nereids_p0/join/test_join_15.groovy b/regression-test/suites/nereids_p0/join/test_join_15.groovy
index 537c6e2611efdc4..d46a216d2d2cf31 100644
--- a/regression-test/suites/nereids_p0/join/test_join_15.groovy
+++ b/regression-test/suites/nereids_p0/join/test_join_15.groovy
@@ -18,7 +18,7 @@
 suite("test_join_15", "nereids_p0") {
     sql "SET enable_nereids_planner=true"
     sql "SET enable_fallback_to_original_planner=false"
-    sql 'set parallel_fragment_exec_instance_num = 2;'
+    sql 'set parallel_pipeline_task_num = 2;'
     sql "use nereids_test_query_db"
 
     def tbName1 = "test"
diff --git a/regression-test/suites/nereids_p0/limit/sql/useRestrictiveLimitFromSubq.sql b/regression-test/suites/nereids_p0/limit/sql/useRestrictiveLimitFromSubq.sql
index b538756492950e1..cf350fc41148f6d 100644
--- a/regression-test/suites/nereids_p0/limit/sql/useRestrictiveLimitFromSubq.sql
+++ b/regression-test/suites/nereids_p0/limit/sql/useRestrictiveLimitFromSubq.sql
@@ -1,3 +1,3 @@
 -- database: presto; groups: limit; tables: nation
-SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=2, parallel_pipeline_task_num=2) */
+SELECT /*+SET_VAR(parallel_pipeline_task_num=2) */
 COUNT(*) FROM (SELECT * FROM tpch_tiny_nation LIMIT 2) AS foo LIMIT 5
diff --git a/regression-test/suites/nereids_p0/limit/sql/withGroupByInSubq.sql b/regression-test/suites/nereids_p0/limit/sql/withGroupByInSubq.sql
index d097ee3c4d68984..85a1689d2ce0d61 100644
--- a/regression-test/suites/nereids_p0/limit/sql/withGroupByInSubq.sql
+++ b/regression-test/suites/nereids_p0/limit/sql/withGroupByInSubq.sql
@@ -1,5 +1,5 @@
 -- database: presto; groups: limit; tables: partsupp
-SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=2, parallel_pipeline_task_num=2) */
+SELECT /*+SET_VAR(parallel_pipeline_task_num=2) */
 COUNT(*) FROM (
     SELECT suppkey, COUNT(*) FROM tpch_tiny_partsupp
     GROUP BY suppkey LIMIT 20) t1
diff --git a/regression-test/suites/nereids_p0/limit/sql/withJoinInSubq.sql b/regression-test/suites/nereids_p0/limit/sql/withJoinInSubq.sql
index be696d9dc030b18..427db0cfc71b23e 100644
--- a/regression-test/suites/nereids_p0/limit/sql/withJoinInSubq.sql
+++ b/regression-test/suites/nereids_p0/limit/sql/withJoinInSubq.sql
@@ -1,2 +1,2 @@
-SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=2, parallel_pipeline_task_num=2) */
+SELECT /*+SET_VAR(parallel_pipeline_task_num=2) */
 COUNT(*) FROM (SELECT n1.regionkey, n1.nationkey FROM tpch_tiny_nation n1 JOIN tpch_tiny_nation n2 ON n1.regionkey = n2.regionkey LIMIT 5) foo
diff --git a/regression-test/suites/nereids_p0/limit/sql/withSubq.sql b/regression-test/suites/nereids_p0/limit/sql/withSubq.sql
index 00ccede952ae933..342aba73dd5fc61 100644
--- a/regression-test/suites/nereids_p0/limit/sql/withSubq.sql
+++ b/regression-test/suites/nereids_p0/limit/sql/withSubq.sql
@@ -1,3 +1,3 @@
 -- database: presto; groups: limit; tables: nation
-SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=2, parallel_pipeline_task_num=2) */
+SELECT /*+SET_VAR(parallel_pipeline_task_num=2) */
 COUNT(*) FROM (SELECT * FROM tpch_tiny_nation LIMIT 10) t1
diff --git a/regression-test/suites/nereids_p0/sort/sort.groovy b/regression-test/suites/nereids_p0/sort/sort.groovy
index 970bf64096cf7d8..3b54ca10be2261b 100644
--- a/regression-test/suites/nereids_p0/sort/sort.groovy
+++ b/regression-test/suites/nereids_p0/sort/sort.groovy
@@ -133,7 +133,7 @@ suite("sort") {
     sb.append("""('2023-03-21 08:00:00', 1.1,1)""")
     sql """ ${sb.toString()} """
 
-    qt_order_by_float """ select /*SET_VAR(parallel_pipeline_task_num=1,parallel_fragment_exec_instance_num=1)*/ * from ${tblName} order by dc; """
-    qt_order_by_int """ select /*SET_VAR(parallel_pipeline_task_num=1,parallel_fragment_exec_instance_num=1)*/ * from ${tblName} order by ic; """
-    qt_order_by_uint """ select /*SET_VAR(parallel_pipeline_task_num=1,parallel_fragment_exec_instance_num=1)*/ * from ${tblName} order by time_period; """
+    qt_order_by_float """ select /*SET_VAR(parallel_pipeline_task_num=1)*/ * from ${tblName} order by dc; """
+    qt_order_by_int """ select /*SET_VAR(parallel_pipeline_task_num=1)*/ * from ${tblName} order by ic; """
+    qt_order_by_uint """ select /*SET_VAR(parallel_pipeline_task_num=1)*/ * from ${tblName} order by time_period; """
 }
diff --git a/regression-test/suites/nereids_p0/sql_functions/aggregate_functions/test_aggregate_retention.sql b/regression-test/suites/nereids_p0/sql_functions/aggregate_functions/test_aggregate_retention.sql
index 93dcaffde714bf3..fa600200a681aa1 100644
--- a/regression-test/suites/nereids_p0/sql_functions/aggregate_functions/test_aggregate_retention.sql
+++ b/regression-test/suites/nereids_p0/sql_functions/aggregate_functions/test_aggregate_retention.sql
@@ -43,7 +43,7 @@ SELECT
             GROUP BY uid 
             ORDER BY uid ASC;
 
-SET parallel_fragment_exec_instance_num=4;
+SET parallel_pipeline_task_num=4;
 
 SELECT * from retention_test ORDER BY uid;
 
diff --git a/regression-test/suites/nereids_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy b/regression-test/suites/nereids_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy
index 636c8465efafaf3..2928490fc50b1b6 100644
--- a/regression-test/suites/nereids_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy
+++ b/regression-test/suites/nereids_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy
@@ -618,8 +618,8 @@ suite("test_bitmap_function") {
     qt_sql """ select orthogonal_bitmap_intersect_count(members, tag_group, 1150000, 1150001, 390006) from ${arthogonalBitmapTable} where  tag_group in ( 1150000, 1150001, 390006); """
     qt_sql """ select orthogonal_bitmap_union_count(members) from ${arthogonalBitmapTable} where  tag_group in ( 1150000, 1150001, 390006);  """
     qt_sql_orthogonal_bitmap_intersect_count2 """ select orthogonal_bitmap_intersect_count(members, tag_group, 1,2) from test_arthogonal_bitmap; """
-    qt_sql_orthogonal_bitmap_intersect_count3_1 """ select /*+SET_VAR(parallel_fragment_exec_instance_num=1, parallel_pipeline_task_num=1)*/orthogonal_bitmap_intersect_count(members, tag_group, 1,11) from test_arthogonal_bitmap; """
-    qt_sql_orthogonal_bitmap_intersect_count3_2 """ select /*+SET_VAR(parallel_fragment_exec_instance_num=2, parallel_pipeline_task_num=2)*/orthogonal_bitmap_intersect_count(members, tag_group, 1,11) from test_arthogonal_bitmap; """
+    qt_sql_orthogonal_bitmap_intersect_count3_1 """ select /*+SET_VAR(parallel_pipeline_task_num=1)*/orthogonal_bitmap_intersect_count(members, tag_group, 1,11) from test_arthogonal_bitmap; """
+    qt_sql_orthogonal_bitmap_intersect_count3_2 """ select /*+SET_VAR(parallel_pipeline_task_num=2)*/orthogonal_bitmap_intersect_count(members, tag_group, 1,11) from test_arthogonal_bitmap; """
     qt_sql_orthogonal_bitmap_intersect_count4 """ select orthogonal_bitmap_intersect_count(members, tag_group, 2,12) from test_arthogonal_bitmap; """
     qt_sql_orthogonal_bitmap_union_count2 """ select orthogonal_bitmap_union_count( cast(null as bitmap)) from test_arthogonal_bitmap; """
     qt_sql_orthogonal_bitmap_union_count3 """ select orthogonal_bitmap_union_count(members) from test_arthogonal_bitmap; """
diff --git a/regression-test/suites/nereids_p0/sql_functions/conditional_functions/test_nullif.groovy b/regression-test/suites/nereids_p0/sql_functions/conditional_functions/test_nullif.groovy
index 343d74d591bb22b..9223d98dc3b632a 100644
--- a/regression-test/suites/nereids_p0/sql_functions/conditional_functions/test_nullif.groovy
+++ b/regression-test/suites/nereids_p0/sql_functions/conditional_functions/test_nullif.groovy
@@ -99,11 +99,11 @@ suite("test_nullif") {
     qt_if_nullif8 """select ifnull(b.k1, -1) k1 from baseall a left join bigtable b on a.k1 = b.k1 + 5 
             order by a.k1"""
     // make sure stable
-    qt_if_nullif8_1 """select   /*+ SET_VAR(enable_pipeline_engine=false,parallel_fragment_exec_instance_num=2,parallel_pipeline_task_num=2,enable_share_hash_table_for_broadcast_join=true) */ b.k1, ifnull(b.k1, -1) k1 from baseall a left join bigtable b on a.k1 = b.k1 + 5 order by a.k1;"""
-    qt_if_nullif8_2 """select   /*+ SET_VAR(enable_pipeline_engine=false,parallel_fragment_exec_instance_num=2,parallel_pipeline_task_num=2,enable_share_hash_table_for_broadcast_join=true) */ b.k1, ifnull(b.k1, -1) k1 from baseall a left join bigtable b on a.k1 = b.k1 + 5 order by a.k1;"""
-    qt_if_nullif8_3 """select   /*+ SET_VAR(enable_pipeline_engine=false,parallel_fragment_exec_instance_num=2,parallel_pipeline_task_num=2,enable_share_hash_table_for_broadcast_join=true) */ b.k1, ifnull(b.k1, -1) k1 from baseall a left join bigtable b on a.k1 = b.k1 + 5 order by a.k1;"""
-    qt_if_nullif8_4 """select   /*+ SET_VAR(enable_pipeline_engine=false,parallel_fragment_exec_instance_num=2,parallel_pipeline_task_num=2,enable_share_hash_table_for_broadcast_join=true) */ b.k1, ifnull(b.k1, -1) k1 from baseall a left join bigtable b on a.k1 = b.k1 + 5 order by a.k1;"""
-    qt_if_nullif8_5 """select   /*+ SET_VAR(enable_pipeline_engine=false,parallel_fragment_exec_instance_num=2,parallel_pipeline_task_num=2,enable_share_hash_table_for_broadcast_join=true) */ b.k1, ifnull(b.k1, -1) k1 from baseall a left join bigtable b on a.k1 = b.k1 + 5 order by a.k1;"""
+    qt_if_nullif8_1 """select   /*+ SET_VAR(enable_pipeline_engine=false,parallel_pipeline_task_num=2,enable_share_hash_table_for_broadcast_join=true) */ b.k1, ifnull(b.k1, -1) k1 from baseall a left join bigtable b on a.k1 = b.k1 + 5 order by a.k1;"""
+    qt_if_nullif8_2 """select   /*+ SET_VAR(enable_pipeline_engine=false,parallel_pipeline_task_num=2,enable_share_hash_table_for_broadcast_join=true) */ b.k1, ifnull(b.k1, -1) k1 from baseall a left join bigtable b on a.k1 = b.k1 + 5 order by a.k1;"""
+    qt_if_nullif8_3 """select   /*+ SET_VAR(enable_pipeline_engine=false,parallel_pipeline_task_num=2,enable_share_hash_table_for_broadcast_join=true) */ b.k1, ifnull(b.k1, -1) k1 from baseall a left join bigtable b on a.k1 = b.k1 + 5 order by a.k1;"""
+    qt_if_nullif8_4 """select   /*+ SET_VAR(enable_pipeline_engine=false,parallel_pipeline_task_num=2,enable_share_hash_table_for_broadcast_join=true) */ b.k1, ifnull(b.k1, -1) k1 from baseall a left join bigtable b on a.k1 = b.k1 + 5 order by a.k1;"""
+    qt_if_nullif8_5 """select   /*+ SET_VAR(enable_pipeline_engine=false,parallel_pipeline_task_num=2,enable_share_hash_table_for_broadcast_join=true) */ b.k1, ifnull(b.k1, -1) k1 from baseall a left join bigtable b on a.k1 = b.k1 + 5 order by a.k1;"""
     qt_if_nullif10 """select ifnull(b.k6, "hll") k1 from baseall a left join bigtable b on a.k1 = b.k1 + 5 
             order by k1"""
     qt_if_nullif11 """select ifnull(b.k10, "2017-06-06") k1 from baseall a left join bigtable b on 
diff --git a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy
index bdd92166f7e686b..b5d4a027adcb19d 100644
--- a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy
+++ b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy
@@ -121,7 +121,7 @@ suite("test_date_function") {
             (18, "2019-08-08 13:21:03", "Africa/Lusaka", "America/Creston")
     """
 
-    sql "set parallel_fragment_exec_instance_num = 8"
+    sql "set parallel_pipeline_task_num = 8"
 
     qt_sql1 """
         SELECT
diff --git a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_or_datetime_computation_negative.groovy b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_or_datetime_computation_negative.groovy
index 53b7385b1535df7..7d36b4cc2175a62 100644
--- a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_or_datetime_computation_negative.groovy
+++ b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_or_datetime_computation_negative.groovy
@@ -31,7 +31,7 @@ suite("test_date_or_datetime_computation_negative") {
             );"""
 
     sql "set enable_insert_strict = false;"
-    sql "set parallel_fragment_exec_instance_num = 3;"
+    sql "set parallel_pipeline_task_num = 3;"
     sql "set enable_nereids_planner = true;"
 
     sql """INSERT INTO test_date_or_datetime_computation_negative VALUES (1, '0000-01-01', '0000-01-01', '0000-01-01', '0000-01-01', '0000-01-01 00:00:00', '0000-01-01 00:00:00');"""
diff --git a/regression-test/suites/nereids_p0/sql_functions/window_functions/test_window_function.groovy b/regression-test/suites/nereids_p0/sql_functions/window_functions/test_window_function.groovy
index 3b038340c7f19c3..8ec7518255d88b8 100644
--- a/regression-test/suites/nereids_p0/sql_functions/window_functions/test_window_function.groovy
+++ b/regression-test/suites/nereids_p0/sql_functions/window_functions/test_window_function.groovy
@@ -25,7 +25,7 @@ suite("test_window_function") {
     sql """ INSERT INTO ${windowFunctionTable1} VALUES ('JDR',12.86,'2014-10-02 00:00:00','2014-10-02 00:00:00.111111','2014-10-02 00:00:00.111111','2014-10-02 00:00:00.111111'),('JDR',12.89,'2014-10-03 00:00:00','2014-10-03 00:00:00.111111','2014-10-03 00:00:00.111111','2014-10-03 00:00:00.111111'),('JDR',12.94,'2014-10-04 00:00:00','2014-10-04 00:00:00.111111','2014-10-04 00:00:00.111111','2014-10-04 00:00:00.111111'),('JDR',12.55,'2014-10-05 00:00:00','2014-10-05 00:00:00.111111','2014-10-05 00:00:00.111111','2014-10-05 00:00:00.111111'),('JDR',14.03,'2014-10-06 00:00:00','2014-10-06 00:00:00.111111','2014-10-06 00:00:00.111111','2014-10-06 00:00:00.111111'),('JDR',14.75,'2014-10-07 00:00:00','2014-10-07 00:00:00.111111','2014-10-07 00:00:00.111111','2014-10-07 00:00:00.111111'),('JDR',13.98,'2014-10-08 00:00:00','2014-10-08 00:00:00.111111','2014-10-08 00:00:00.111111','2014-10-08 00:00:00.111111') """
 
     qt_sql """
-            SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+            SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                 stock_symbol,
                 closing_date,
                 closing_price,
@@ -39,7 +39,7 @@ suite("test_window_function") {
            """
     // LEAD
     qt_sql """ 
-             SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+             SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                 stock_symbol,
                 closing_date,
                 closing_price,
@@ -54,12 +54,12 @@ suite("test_window_function") {
 
     // LEAD not nullable coredump
     qt_sql """
-           select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ t1.new_time from (select closing_date, lead(closing_date, 1, '2014-10-02 00:00:00') over () as new_time from ${windowFunctionTable1}) as t1 left join ${windowFunctionTable1} t2 on t2.closing_date = t1.closing_date order by t1.new_time desc;
+           select /*+SET_VAR(parallel_pipeline_task_num=1) */ t1.new_time from (select closing_date, lead(closing_date, 1, '2014-10-02 00:00:00') over () as new_time from ${windowFunctionTable1}) as t1 left join ${windowFunctionTable1} t2 on t2.closing_date = t1.closing_date order by t1.new_time desc;
            """
 
     // LAG
     qt_sql """ 
-             SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+             SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                 stock_symbol,
                 closing_date,
                 closing_price,
@@ -71,7 +71,7 @@ suite("test_window_function") {
             """
 
     qt_sql """
-            SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+            SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                 stock_symbol,
                 closing_date1,
                 closing_price,
@@ -85,7 +85,7 @@ suite("test_window_function") {
            """
     // LEAD
     qt_sql """
-             SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+             SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                 stock_symbol,
                 closing_date1,
                 closing_price,
@@ -100,12 +100,12 @@ suite("test_window_function") {
 
     // LEAD not nullable coredump
     qt_sql """  
-           select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ t1.new_time from (select closing_date1, lead(closing_date1, 1, '2014-10-02 00:00:00') over () as new_time from ${windowFunctionTable1}) as t1 left join ${windowFunctionTable1} t2 on t2.closing_date1 = t1.closing_date1 order by t1.new_time desc;
+           select /*+SET_VAR(parallel_pipeline_task_num=1) */ t1.new_time from (select closing_date1, lead(closing_date1, 1, '2014-10-02 00:00:00') over () as new_time from ${windowFunctionTable1}) as t1 left join ${windowFunctionTable1} t2 on t2.closing_date1 = t1.closing_date1 order by t1.new_time desc;
            """
 
     // LAG
     qt_sql """
-             SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+             SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                 stock_symbol,
                 closing_date1,
                 closing_price,
@@ -117,7 +117,7 @@ suite("test_window_function") {
             """
 
     qt_sql """
-            SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+            SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                 stock_symbol,
                 closing_date2,
                 closing_price,
@@ -131,7 +131,7 @@ suite("test_window_function") {
            """
     // LEAD
     qt_sql """
-             SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+             SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                 stock_symbol,
                 closing_date2,
                 closing_price,
@@ -146,12 +146,12 @@ suite("test_window_function") {
 
     // LEAD not nullable coredump
     qt_sql """
-           select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ t1.new_time from (select closing_date2, lead(closing_date2, 1, '2014-10-02 00:00:00') over () as new_time from ${windowFunctionTable1}) as t1 left join ${windowFunctionTable1} t2 on t2.closing_date2 = t1.closing_date2 order by t1.new_time desc;
+           select /*+SET_VAR(parallel_pipeline_task_num=1) */ t1.new_time from (select closing_date2, lead(closing_date2, 1, '2014-10-02 00:00:00') over () as new_time from ${windowFunctionTable1}) as t1 left join ${windowFunctionTable1} t2 on t2.closing_date2 = t1.closing_date2 order by t1.new_time desc;
            """
 
     // LAG
     qt_sql """
-             SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+             SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                 stock_symbol,
                 closing_date2,
                 closing_price,
@@ -163,7 +163,7 @@ suite("test_window_function") {
             """
 
     qt_sql """
-            SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+            SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                 stock_symbol,
                 closing_date3,
                 closing_price,
@@ -177,7 +177,7 @@ suite("test_window_function") {
            """
     // LEAD
     qt_sql """
-             SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+             SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                 stock_symbol,
                 closing_date3,
                 closing_price,
@@ -192,12 +192,12 @@ suite("test_window_function") {
 
     // LEAD not nullable coredump
     qt_sql """
-           select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ t1.new_time from (select closing_date3, lead(closing_date3, 1, '2014-10-02 00:00:00') over () as new_time from ${windowFunctionTable1}) as t1 left join ${windowFunctionTable1} t2 on t2.closing_date3 = t1.closing_date3 order by t1.new_time desc;
+           select /*+SET_VAR(parallel_pipeline_task_num=1) */ t1.new_time from (select closing_date3, lead(closing_date3, 1, '2014-10-02 00:00:00') over () as new_time from ${windowFunctionTable1}) as t1 left join ${windowFunctionTable1} t2 on t2.closing_date3 = t1.closing_date3 order by t1.new_time desc;
            """
 
     // LAG
     qt_sql """
-             SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+             SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                 stock_symbol,
                 closing_date3,
                 closing_price,
@@ -217,7 +217,7 @@ suite("test_window_function") {
 
     // SUM
     qt_sql """
-               SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+               SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                     x,
                     property,
                     sum( x ) over ( PARTITION BY property ORDER BY x rows BETWEEN 1 preceding AND 1 following ) AS 'moving total' 
@@ -230,7 +230,7 @@ suite("test_window_function") {
            """
     // AVG
     qt_sql """
-               SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+               SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                     x,
                     property,
                     avg( x ) over ( PARTITION BY property ORDER BY x rows BETWEEN 1 preceding AND 1 following ) AS 'moving average' 
@@ -243,7 +243,7 @@ suite("test_window_function") {
            """
     // COUNT
     qt_sql """
-               SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+               SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                     x,
                     property,
                     count( x ) over ( PARTITION BY property ORDER BY x rows BETWEEN unbounded preceding AND current ROW ) AS 'cumulative total' 
@@ -259,7 +259,7 @@ suite("test_window_function") {
 
     // MIN
     qt_sql """ 
-              SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+              SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                    x,
                    property,
                    min( x ) over ( ORDER BY property, x DESC rows BETWEEN unbounded preceding AND 1 following ) AS 'local minimum' 
@@ -270,7 +270,7 @@ suite("test_window_function") {
            """
     // MAX
     qt_sql """
-              SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+              SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                    x,
                    property,
                    max( x ) over ( ORDER BY property, x rows BETWEEN unbounded preceding AND 1 following ) AS 'local maximum' 
@@ -288,15 +288,15 @@ suite("test_window_function") {
     sql """ insert into  ${windowFunctionTable3} values (1,1),(1,2),(1,2),(2,1),(2,2),(2,3),(3,1),(3,1),(3,2); """
 
     // RANK
-    qt_sql """ select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ x, y, rank() over(partition by x order by y) as rank from ${windowFunctionTable3} order by x, y; """
+    qt_sql """ select /*+SET_VAR(parallel_pipeline_task_num=1) */ x, y, rank() over(partition by x order by y) as rank from ${windowFunctionTable3} order by x, y; """
     // DENSE_RANK
-    qt_sql """ select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ x, y, dense_rank() over(partition by x order by y) as rank from ${windowFunctionTable3} order by x, y; """
+    qt_sql """ select /*+SET_VAR(parallel_pipeline_task_num=1) */ x, y, dense_rank() over(partition by x order by y) as rank from ${windowFunctionTable3} order by x, y; """
     // PERCENT_RANK
-    qt_sql """ select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ x, y, percent_rank() over(partition by x order by y) as rank from ${windowFunctionTable3} order by x, y; """
+    qt_sql """ select /*+SET_VAR(parallel_pipeline_task_num=1) */ x, y, percent_rank() over(partition by x order by y) as rank from ${windowFunctionTable3} order by x, y; """
     // CUME_DIST
-    qt_sql """ select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ x, y, cume_dist() over(partition by x order by y) as rank from ${windowFunctionTable3} order by x, y; """
+    qt_sql """ select /*+SET_VAR(parallel_pipeline_task_num=1) */ x, y, cume_dist() over(partition by x order by y) as rank from ${windowFunctionTable3} order by x, y; """
     // ROW_NUMBER
-    qt_sql """ select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ x, y, row_number() over(partition by x order by y) as rank from ${windowFunctionTable3} order by x, y; """
+    qt_sql """ select /*+SET_VAR(parallel_pipeline_task_num=1) */ x, y, row_number() over(partition by x order by y) as rank from ${windowFunctionTable3} order by x, y; """
 
     sql """ drop table   ${windowFunctionTable3}  """
 
@@ -307,9 +307,9 @@ suite("test_window_function") {
     sql """ insert into ${windowFunctionTable4} VALUES ('Pete','USA','Hello'),('John','USA','Hi'),('Boris','Germany','Guten tag'),('Michael','Germany','Guten morgen'),('Bjorn','Sweden','Hej'),('Mats','Sweden','Tja')"""
 
     // first_value
-    qt_sql """ select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ country, name,first_value(greeting) over (partition by country order by name, greeting) as greeting from ${windowFunctionTable4} order by country, name; """
+    qt_sql """ select /*+SET_VAR(parallel_pipeline_task_num=1) */ country, name,first_value(greeting) over (partition by country order by name, greeting) as greeting from ${windowFunctionTable4} order by country, name; """
     // last_value
-    qt_sql """ select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ country, name,last_value(greeting)  over (partition by country order by name, greeting) as greeting from ${windowFunctionTable4} order by country, name; """
+    qt_sql """ select /*+SET_VAR(parallel_pipeline_task_num=1) */ country, name,last_value(greeting)  over (partition by country order by name, greeting) as greeting from ${windowFunctionTable4} order by country, name; """
 
     sql """ drop table   ${windowFunctionTable4}  """
 
@@ -320,15 +320,15 @@ suite("test_window_function") {
     String k1 = fields[3]
     String k2 = fields[5]
     String k3 = fields[3]
-    qt_first_value1"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, first_value(${k2}) over (partition by ${k1} order by ${k3}, ${k2})
+    qt_first_value1"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, first_value(${k2}) over (partition by ${k1} order by ${k3}, ${k2})
              as wj from baseall  order by ${k1}, wj"""
-    qt_first_value2"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, first_value(${k2}) over (partition by ${k1} order by ${k3}, ${k2}
+    qt_first_value2"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, first_value(${k2}) over (partition by ${k1} order by ${k3}, ${k2}
              range between unbounded preceding and current row)
              as wj from baseall  order by ${k1}, wj"""
-    qt_first_value3"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, first_value(${k2}) over (partition by ${k1} order by ${k3}, ${k2} 
+    qt_first_value3"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, first_value(${k2}) over (partition by ${k1} order by ${k3}, ${k2}
              rows between unbounded preceding and current row)
              as wj from baseall  order by ${k1}, wj"""
-    qt_first_value4"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ a, min(d) as wjj from 
+    qt_first_value4"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ a, min(d) as wjj from
              (select t1.k1 as k1, t1.k2 as k2, t1.k3 as k3,
              t1.k4 as k4, t1.k5 as k5,t1.k6 as k6,
              t1.k7 as k7, t1.k8 as k8, t1.k9 as k9,
@@ -340,15 +340,15 @@ suite("test_window_function") {
              order by a, wjj"""
 
     // test_query_last_value
-    qt_last_value1"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, last_value(${k2}) over (partition by ${k1} order by ${k3},${k2})
+    qt_last_value1"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, last_value(${k2}) over (partition by ${k1} order by ${k3},${k2})
                  as wj from baseall  order by ${k1}, wj"""
-    qt_last_value2"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, last_value(${k2}) over (partition by ${k1} order by ${k3},${k2}
+    qt_last_value2"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, last_value(${k2}) over (partition by ${k1} order by ${k3},${k2}
              range between unbounded preceding and current row)
              as wj from baseall  order by ${k1}, wj"""
-    qt_last_value3"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, last_value(${k2}) over (partition by ${k1} order by ${k3},${k2}
+    qt_last_value3"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, last_value(${k2}) over (partition by ${k1} order by ${k3},${k2}
              rows between unbounded preceding and current row)
              as wj from baseall  order by ${k1}, wj"""
-    qt_last_value4"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ a, max(d) as wjj from 
+    qt_last_value4"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ a, max(d) as wjj from
              (select t1.k1 as k1, t1.k2 as k2, t1.k3 as k3,
              t1.k4 as k4, t1.k5 as k5,t1.k6 as k6,
              t1.k7 as k7, t1.k8 as k8, t1.k9 as k9,
@@ -360,79 +360,79 @@ suite("test_window_function") {
              order by a, wjj"""
 
     // test_query_row_number
-    qt_row_number1"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, row_number() over (partition by ${k1} order by ${k3}) 
+    qt_row_number1"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, row_number() over (partition by ${k1} order by ${k3})
                     as wj from baseall order by ${k1}, wj"""
-    qt_row_number2"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, count(k1) over (partition by ${k1} order by ${k3}
+    qt_row_number2"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, count(k1) over (partition by ${k1} order by ${k3}
                     rows between unbounded preceding and current row)
                     as wj from baseall order by ${k1}, wj"""
 
     // test error
     test {
-        sql("select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, lag(${k2}) over (partition by ${k1} order by ${k3}) from baseall")
+        sql("select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, lag(${k2}) over (partition by ${k1} order by ${k3}) from baseall")
         exception ""
     }
     test {
-        sql"select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, lag(${k2}, -1, 1) over (partition by ${k1} order by ${k3}) from baseall"
+        sql"select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, lag(${k2}, -1, 1) over (partition by ${k1} order by ${k3}) from baseall"
         exception ""
     }
     test {
-        sql"select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, lag(${k2}, 1) over (partition by ${k1} order by ${k3}) from baseall"
+        sql"select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, lag(${k2}, 1) over (partition by ${k1} order by ${k3}) from baseall"
         exception ""
     }
     test {
-        sql"select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, lead(${k2}) over (partition by ${k1} order by ${k3}) from baseall"
+        sql"select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, lead(${k2}) over (partition by ${k1} order by ${k3}) from baseall"
         exception ""
     }
     test {
-        sql"select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, lead(${k2}, -1, 1) over (partition by ${k1} order by ${k3}) from baseall"
+        sql"select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, lead(${k2}, -1, 1) over (partition by ${k1} order by ${k3}) from baseall"
         exception ""
     }
     test {
-        sql"select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, lead(${k2}, 1) over (partition by ${k1} order by ${k3}) from baseall"
+        sql"select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, lead(${k2}, 1) over (partition by ${k1} order by ${k3}) from baseall"
         exception ""
     }
-    qt_window_error1"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, first_value(${k2}) over (partition by ${k1}) from baseall order by ${k1}"""
-    qt_window_error2"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, first_value(${k2}) over (order by ${k3}) from baseall"""
-    qt_window_error3"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, max(${k2}) over (order by ${k3}) from baseall"""
+    qt_window_error1"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, first_value(${k2}) over (partition by ${k1}) from baseall order by ${k1}"""
+    qt_window_error2"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, first_value(${k2}) over (order by ${k3}) from baseall"""
+    qt_window_error3"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, max(${k2}) over (order by ${k3}) from baseall"""
     test {
-        sql"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, sum(${k2}) over (partition by ${k1} order by ${k3} rows
+        sql"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, sum(${k2}) over (partition by ${k1} order by ${k3} rows
             between current row and unbounded preceding) as wj
             from baseall order by ${k1}, wj"""
         exception ""
     }
     test {
-        sql"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, sum(${k2}) over (partition by ${k1} order by ${k3} rows 
+        sql"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, sum(${k2}) over (partition by ${k1} order by ${k3} rows
             between 0 preceding and 1 following) as wj 
             from baseall order by ${k1}, wj"""
         exception ""
     }
     test {
-        sql"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, sum(${k2}) over (partition by ${k1} order by ${k3} rows 
+        sql"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, sum(${k2}) over (partition by ${k1} order by ${k3} rows
             between unbounded following and current row) as wj 
             from baseall order by ${k1}, wj"""
         exception ""
     }
     test {
-        sql"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, rank(${k2}) over (partition by ${k1} order by ${k3}) as wj
+        sql"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, rank(${k2}) over (partition by ${k1} order by ${k3}) as wj
             from baseall order by ${k1}, wj"""
         exception ""
     }
     test {
-        sql"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, max() over (partition by ${k1} order by ${k3}) as wj
+        sql"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, max() over (partition by ${k1} order by ${k3}) as wj
             from baseall order by ${k1}, wj"""
         exception ""
     }
     test {
-        sql"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, count(${k2}) over (order by ${k1} rows partition by ${k3}) as wj
+        sql"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, count(${k2}) over (order by ${k1} rows partition by ${k3}) as wj
             from baseall order by ${k1}, wj"""
         exception ""
     }
 
     // test_query_rank
     k3 = fields[7]
-    qt_rank1"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, rank() over (partition by ${k1} order by ${k3}) as wj 
+    qt_rank1"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, rank() over (partition by ${k1} order by ${k3}) as wj
              from baseall order by ${k1}, wj"""
-    qt_rank2"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ F2.${k1}, (F1.wj - F2.basewj + 1) as wj from
+    qt_rank2"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ F2.${k1}, (F1.wj - F2.basewj + 1) as wj from
              (select a, c, count(*) as wj from 
              (select t1.k1 as k1, t1.k2 as k2, t1.k3 as k3,
              t1.k4 as k4, t1.k5 as k5,t1.k6 as k6,
@@ -446,16 +446,16 @@ suite("test_window_function") {
              where F1.a=F2.${k1} and F1.c = F2.${k3} order by F2.${k1}, wj"""
 
     //test_hang
-    qt_window_hang1"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, row_number() over (partition by ${k1} order by ${k3}) as wj from
+    qt_window_hang1"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, row_number() over (partition by ${k1} order by ${k3}) as wj from
              baseall order by ${k1}, wj"""
     String line = "("
     String cur
     for (p in range(0, 829)) {
         if (p == 0) {
-            cur = "(select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, 1 as wj from baseall order by ${k1}, ${k3} limit 1)".toString()
+            cur = "(select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, 1 as wj from baseall order by ${k1}, ${k3} limit 1)".toString()
         }
         else {
-            cur = """(select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, ${p+1} as wj from baseall order by ${k1} , ${k3}
+            cur = """(select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, ${p+1} as wj from baseall order by ${k1} , ${k3}
                     limit ${p}, 1 ) """.toString()
 
         }
@@ -469,7 +469,7 @@ suite("test_window_function") {
 
     sql """ admin set frontend config("remote_fragment_exec_timeout_ms"="300000"); """
 
-    //qt_window_hang2"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ A.${k1}, A.wj - B.dyk + 1 as num from
+    //qt_window_hang2"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ A.${k1}, A.wj - B.dyk + 1 as num from
     //    (select ${k1}, wj from ${line} as W1) as A join
     //    (select ${k1}, min(wj) as dyk from ${line} as W2 group by ${k1}) as B
     //    where A.${k1}=B.${k1}  order by A.${k1}, num"""
@@ -478,9 +478,9 @@ suite("test_window_function") {
     line = "("
     for (p in range(0, 829)) {
         if (p == 0 ) {
-            cur = "(select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ * from baseall order by k1, k6 limit 1)"
+            cur = "(select /*+SET_VAR(parallel_pipeline_task_num=1) */ * from baseall order by k1, k6 limit 1)"
         } else {
-            cur = "(select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ * from baseall order by k1, k6 limit ${p}, 1)"
+            cur = "(select /*+SET_VAR(parallel_pipeline_task_num=1) */ * from baseall order by k1, k6 limit ${p}, 1)"
         }
         if (p < 828) {
             line = line + cur + " union all "
@@ -489,12 +489,12 @@ suite("test_window_function") {
         }
     }
     // qt_hujie1"select T.k1, T.k6 from ${line} as T order by T.k1, T.k6"
-    qt_hujie2"select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ k1, k6 from baseall order by k1, k6"
+    qt_hujie2"select /*+SET_VAR(parallel_pipeline_task_num=1) */ k1, k6 from baseall order by k1, k6"
 
     // test_bug
-    order_qt_window_bug1"""SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ wj FROM (SELECT row_number() over (PARTITION BY k6 ORDER BY k1) AS wj 
+    order_qt_window_bug1"""SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */ wj FROM (SELECT row_number() over (PARTITION BY k6 ORDER BY k1) AS wj
             FROM baseall ) AS A where wj = 2"""
-    order_qt_window_bug2"""SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ A.k2 AS a, A.k1 as b, B.k1 as c, B.k2 as d FROM  
+    order_qt_window_bug2"""SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */ A.k2 AS a, A.k1 as b, B.k1 as c, B.k2 as d FROM
            ( SELECT k2, k1, row_number () over (PARTITION BY k2 ORDER BY k3) AS wj  
            FROM baseall ) AS A JOIN ( SELECT k2, k1, row_number () over  
            (PARTITION BY k2 ORDER BY k3) AS wj FROM baseall ) AS B WHERE A.k2=B.k2"""
diff --git a/regression-test/suites/nereids_rules_p0/cte/test_cte_filter_pushdown.groovy b/regression-test/suites/nereids_rules_p0/cte/test_cte_filter_pushdown.groovy
index 258e220a4a65d06..4f28614fdd70b89 100644
--- a/regression-test/suites/nereids_rules_p0/cte/test_cte_filter_pushdown.groovy
+++ b/regression-test/suites/nereids_rules_p0/cte/test_cte_filter_pushdown.groovy
@@ -87,7 +87,6 @@ suite("test_cte_filter_pushdown") {
     """
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy b/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy
index 942c4ce88dee7a9..878fb3801ed62da 100644
--- a/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy
@@ -574,7 +574,7 @@ suite("variant_mv") {
     where g2.actor['id'] > 34259289;
     """
     def query3_6 = """
-    SELECT  /*+SET_VAR(batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=3,parallel_pipeline_task_num=0,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=true,parallel_scan_max_scanners_count=32,parallel_scan_min_rows_per_scanner=64,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=1,enable_parallel_result_sink=false,enable_nereids_planner=true,rewrite_or_to_in_predicate_threshold=100000,enable_function_pushdown=false,enable_common_expr_pushdown=false,enable_local_exchange=true,enable_share_hash_table_for_broadcast_join=true,enable_two_phase_read_opt=true,enable_common_expr_pushdown_for_inverted_index=false,enable_delete_sub_predicate_v2=false,min_revocable_mem=33554432,fetch_remote_schema_timeout_seconds=120,max_fetch_remote_schema_tablet_count=512,enable_join_spill=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5) */
+    SELECT  /*+SET_VAR(batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_pipeline_task_num=0,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=true,parallel_scan_max_scanners_count=32,parallel_scan_min_rows_per_scanner=64,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=1,enable_parallel_result_sink=false,enable_nereids_planner=true,rewrite_or_to_in_predicate_threshold=100000,enable_function_pushdown=false,enable_common_expr_pushdown=false,enable_local_exchange=true,enable_share_hash_table_for_broadcast_join=true,enable_two_phase_read_opt=true,enable_common_expr_pushdown_for_inverted_index=false,enable_delete_sub_predicate_v2=false,min_revocable_mem=33554432,fetch_remote_schema_timeout_seconds=120,max_fetch_remote_schema_tablet_count=512,enable_join_spill=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5) */
     g1.id,
     g2.type,
     floor(cast(g1.actor['id'] as int) + 100.5),
diff --git a/regression-test/suites/nereids_syntax_p0/group_by_constant.groovy b/regression-test/suites/nereids_syntax_p0/group_by_constant.groovy
index 30a24e5fdb8de34..f65392c34a8169b 100644
--- a/regression-test/suites/nereids_syntax_p0/group_by_constant.groovy
+++ b/regression-test/suites/nereids_syntax_p0/group_by_constant.groovy
@@ -19,7 +19,7 @@ suite("group_by_constant") {
     sql """
         SET enable_nereids_planner=true
     """
-    sql "set parallel_fragment_exec_instance_num=8"
+    sql "set parallel_pipeline_task_num=8"
     sql "SET enable_fallback_to_original_planner=false"
 
     qt_select_1 """ 
diff --git a/regression-test/suites/nereids_syntax_p0/null_aware_left_anti_join.groovy b/regression-test/suites/nereids_syntax_p0/null_aware_left_anti_join.groovy
index c09358073f98e25..edb65b5114a47e7 100644
--- a/regression-test/suites/nereids_syntax_p0/null_aware_left_anti_join.groovy
+++ b/regression-test/suites/nereids_syntax_p0/null_aware_left_anti_join.groovy
@@ -56,7 +56,7 @@ suite("test_nereids_null_aware_left_anti_join") {
 
     qt_select """ select ${tableName2}.k1 from ${tableName2} where k1 not in (select ${tableName1}.k1 from ${tableName1}) order by ${tableName2}.k1; """
 
-    sql """ set parallel_fragment_exec_instance_num=2; """
+    sql """ set parallel_pipeline_task_num=2; """
     sql """ set parallel_pipeline_task_num=2; """
     qt_select """ select ${tableName2}.k1 from ${tableName2} where k1 not in (select ${tableName1}.k1 from ${tableName1}) order by ${tableName2}.k1; """
 
diff --git a/regression-test/suites/nereids_syntax_p0/scan_partition.groovy b/regression-test/suites/nereids_syntax_p0/scan_partition.groovy
index bbe9459300c19ee..e25c0becb948bd3 100644
--- a/regression-test/suites/nereids_syntax_p0/scan_partition.groovy
+++ b/regression-test/suites/nereids_syntax_p0/scan_partition.groovy
@@ -43,7 +43,7 @@ suite("test_scan_part") {
     sql "SET enable_nereids_planner=true"
     sql "SET enable_fallback_to_original_planner=false"
 
-    sql "SET parallel_fragment_exec_instance_num=8"
+    sql "SET parallel_pipeline_task_num=8"
     qt_sql """
             SELECT id, SUM(i1) FROM test_part WHERE i2 IN ('1','5')
            GROUP BY id HAVING id = 1;
diff --git a/regression-test/suites/nereids_tpch_p0/tpch/q1.groovy b/regression-test/suites/nereids_tpch_p0/tpch/q1.groovy
index 373dc37791b5f6e..d7efa248bd2fdb4 100644
--- a/regression-test/suites/nereids_tpch_p0/tpch/q1.groovy
+++ b/regression-test/suites/nereids_tpch_p0/tpch/q1.groovy
@@ -48,7 +48,7 @@ suite("nereids_tpch_q1") {
     """
 
     qt_select """
-    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=16, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=false, enable_projection=false) */
+    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=16, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=false, enable_projection=false) */
         l_returnflag,
         l_linestatus,
         sum(l_quantity) as sum_qty,
diff --git a/regression-test/suites/nereids_tpch_p0/tpch/q10.groovy b/regression-test/suites/nereids_tpch_p0/tpch/q10.groovy
index 806d58f9119ae3c..5b19cf3843100f1 100644
--- a/regression-test/suites/nereids_tpch_p0/tpch/q10.groovy
+++ b/regression-test/suites/nereids_tpch_p0/tpch/q10.groovy
@@ -60,7 +60,7 @@ suite("nereids_tpch_q10") {
     """
 
     qt_select """
-    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=16, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=false, enable_projection=true) */
+    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=16, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=false, enable_projection=true) */
         c_custkey,
         c_name,
         sum(t1.l_extendedprice * (1 - t1.l_discount)) as revenue,
diff --git a/regression-test/suites/nereids_tpch_p0/tpch/q11.groovy b/regression-test/suites/nereids_tpch_p0/tpch/q11.groovy
index 26800974e691a52..0d3f9f157d89d44 100644
--- a/regression-test/suites/nereids_tpch_p0/tpch/q11.groovy
+++ b/regression-test/suites/nereids_tpch_p0/tpch/q11.groovy
@@ -56,7 +56,7 @@ suite("nereids_tpch_q11") {
     """
 
     qt_select """
-    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=2, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=true, enable_projection=true) */
+    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=2, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=true, enable_projection=true) */
         ps_partkey,
         sum(ps_supplycost * ps_availqty) as value
     from
diff --git a/regression-test/suites/nereids_tpch_p0/tpch/q12.groovy b/regression-test/suites/nereids_tpch_p0/tpch/q12.groovy
index 21abd944b763740..77ac34a004872db 100644
--- a/regression-test/suites/nereids_tpch_p0/tpch/q12.groovy
+++ b/regression-test/suites/nereids_tpch_p0/tpch/q12.groovy
@@ -56,7 +56,7 @@ suite("nereids_tpch_q12") {
     """
 
     qt_select """
-    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=2, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=true, enable_projection=true) */
+    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=2, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=true, enable_projection=true) */
         l_shipmode,
         sum(case
             when o_orderpriority = '1-URGENT'
diff --git a/regression-test/suites/nereids_tpch_p0/tpch/q13.groovy b/regression-test/suites/nereids_tpch_p0/tpch/q13.groovy
index 98aa74ac1164ca1..9f5ce9650643136 100644
--- a/regression-test/suites/nereids_tpch_p0/tpch/q13.groovy
+++ b/regression-test/suites/nereids_tpch_p0/tpch/q13.groovy
@@ -48,7 +48,7 @@ suite("nereids_tpch_q13") {
         """
 
         qt_select """
-    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=16, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=true, enable_projection=true) */
+    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=16, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=true, enable_projection=true) */
         c_count,
         count(*) as custdist
     from
diff --git a/regression-test/suites/nereids_tpch_p0/tpch/q14.groovy b/regression-test/suites/nereids_tpch_p0/tpch/q14.groovy
index a2d970352fac3b5..b3ef75770e9dedc 100644
--- a/regression-test/suites/nereids_tpch_p0/tpch/q14.groovy
+++ b/regression-test/suites/nereids_tpch_p0/tpch/q14.groovy
@@ -41,7 +41,7 @@ where
     """
 
     qt_select """
-select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=8, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=true, enable_projection=true) */
+select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=8, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=true, enable_projection=true) */
     100.00 * sum(case
         when p_type like 'PROMO%'
             then l_extendedprice * (1 - l_discount)
diff --git a/regression-test/suites/nereids_tpch_p0/tpch/q15.groovy b/regression-test/suites/nereids_tpch_p0/tpch/q15.groovy
index f71e0b57b712bd3..aef38646f641031 100644
--- a/regression-test/suites/nereids_tpch_p0/tpch/q15.groovy
+++ b/regression-test/suites/nereids_tpch_p0/tpch/q15.groovy
@@ -67,7 +67,7 @@ suite("nereids_tpch_q15") {
     """
 
     qt_select """
-    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=8, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=true, enable_projection=true) */
+    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=8, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=true, enable_projection=true) */
         s_suppkey,
         s_name,
         s_address,
diff --git a/regression-test/suites/nereids_tpch_p0/tpch/q16.groovy b/regression-test/suites/nereids_tpch_p0/tpch/q16.groovy
index 49e02bc41fa20aa..debf7d202ad48fe 100644
--- a/regression-test/suites/nereids_tpch_p0/tpch/q16.groovy
+++ b/regression-test/suites/nereids_tpch_p0/tpch/q16.groovy
@@ -58,7 +58,7 @@ order by
     """
 
     qt_select """
-select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=8, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=true, enable_projection=true) */
+select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=8, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=true, enable_projection=true) */
     p_brand,
     p_type,
     p_size,
diff --git a/regression-test/suites/nereids_tpch_p0/tpch/q17.groovy b/regression-test/suites/nereids_tpch_p0/tpch/q17.groovy
index a391baaa04fd469..9ca3a362494697e 100644
--- a/regression-test/suites/nereids_tpch_p0/tpch/q17.groovy
+++ b/regression-test/suites/nereids_tpch_p0/tpch/q17.groovy
@@ -46,7 +46,7 @@ suite("nereids_tpch_q17") {
     """
 
     qt_select """
-    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=1, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=true, enable_projection=true) */
+    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=1, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=true, enable_projection=true) */
         sum(l_extendedprice) / 7.0 as avg_yearly
     from
         lineitem join
diff --git a/regression-test/suites/nereids_tpch_p0/tpch/q18.groovy b/regression-test/suites/nereids_tpch_p0/tpch/q18.groovy
index 2f2638cea530534..f020bd51b6622a1 100644
--- a/regression-test/suites/nereids_tpch_p0/tpch/q18.groovy
+++ b/regression-test/suites/nereids_tpch_p0/tpch/q18.groovy
@@ -62,7 +62,7 @@ limit 100;
     """
 
      qt_select """
-select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=16, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=true, enable_projection=true) */
+select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=16, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=true, enable_projection=true) */
     c_name,
     c_custkey,
     t3.o_orderkey,
diff --git a/regression-test/suites/nereids_tpch_p0/tpch/q19.groovy b/regression-test/suites/nereids_tpch_p0/tpch/q19.groovy
index 2fb91d16d629c81..aeb02bae2433e40 100644
--- a/regression-test/suites/nereids_tpch_p0/tpch/q19.groovy
+++ b/regression-test/suites/nereids_tpch_p0/tpch/q19.groovy
@@ -64,7 +64,7 @@ where
     """
 
     qt_select """
-select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=2, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=false, enable_projection=true) */
+select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=2, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=false, enable_projection=true) */
     sum(l_extendedprice* (1 - l_discount)) as revenue
 from
     lineitem,
diff --git a/regression-test/suites/nereids_tpch_p0/tpch/q20.groovy b/regression-test/suites/nereids_tpch_p0/tpch/q20.groovy
index 9b2809a1ff6e4a6..b51ed4ae5a3a71d 100644
--- a/regression-test/suites/nereids_tpch_p0/tpch/q20.groovy
+++ b/regression-test/suites/nereids_tpch_p0/tpch/q20.groovy
@@ -66,7 +66,7 @@ order by
     """
 
     qt_select """
-select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=16, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=true, enable_projection=true) */
+select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=16, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=true, enable_projection=true) */
 s_name, s_address from
 supplier left semi join
 (
diff --git a/regression-test/suites/nereids_tpch_p0/tpch/q21.groovy b/regression-test/suites/nereids_tpch_p0/tpch/q21.groovy
index 50ed90a99a61e7a..6e9465c643c4752 100644
--- a/regression-test/suites/nereids_tpch_p0/tpch/q21.groovy
+++ b/regression-test/suites/nereids_tpch_p0/tpch/q21.groovy
@@ -68,7 +68,7 @@ limit 100;
     """
 
     qt_select """
-select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=16, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=true, enable_projection=true) */
+select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=16, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=true, enable_projection=true) */
 s_name, count(*) as numwait
 from orders join
 (
diff --git a/regression-test/suites/nereids_tpch_p0/tpch/q22.groovy b/regression-test/suites/nereids_tpch_p0/tpch/q22.groovy
index 9939491eaf71a6f..fd51ec24738e9db 100644
--- a/regression-test/suites/nereids_tpch_p0/tpch/q22.groovy
+++ b/regression-test/suites/nereids_tpch_p0/tpch/q22.groovy
@@ -65,7 +65,7 @@ order by
     """
 
     qt_select """
-select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=16, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=true, enable_projection=true) */
+select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=16, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=true, enable_projection=true) */
     cntrycode,
     count(*) as numcust,
     sum(c_acctbal) as totacctbal
diff --git a/regression-test/suites/nereids_tpch_p0/tpch/q3.groovy b/regression-test/suites/nereids_tpch_p0/tpch/q3.groovy
index 84f3b576e145fd5..9b5bdbc4241ce34 100644
--- a/regression-test/suites/nereids_tpch_p0/tpch/q3.groovy
+++ b/regression-test/suites/nereids_tpch_p0/tpch/q3.groovy
@@ -51,7 +51,7 @@ suite("nereids_tpch_q3") {
     """
 
     qt_select """
-    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=8, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=false, enable_projection=true) */
+    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=8, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=false, enable_projection=true) */
         l_orderkey,
         sum(l_extendedprice * (1 - l_discount)) as revenue,
         o_orderdate,
diff --git a/regression-test/suites/nereids_tpch_p0/tpch/q4.groovy b/regression-test/suites/nereids_tpch_p0/tpch/q4.groovy
index 6b67065542c523b..bf4fe91d65ec26e 100644
--- a/regression-test/suites/nereids_tpch_p0/tpch/q4.groovy
+++ b/regression-test/suites/nereids_tpch_p0/tpch/q4.groovy
@@ -49,7 +49,7 @@ suite("nereids_tpch_q4") {
     """
 
     qt_select """
-    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=4, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=false, enable_projection=true) */
+    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=4, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=false, enable_projection=true) */
         o_orderpriority,
         count(*) as order_count
     from
diff --git a/regression-test/suites/nereids_tpch_p0/tpch/q6.groovy b/regression-test/suites/nereids_tpch_p0/tpch/q6.groovy
index 758d842007c4fea..4de94068307471d 100644
--- a/regression-test/suites/nereids_tpch_p0/tpch/q6.groovy
+++ b/regression-test/suites/nereids_tpch_p0/tpch/q6.groovy
@@ -37,7 +37,7 @@ suite("nereids_tpch_q6") {
     """
 
     qt_select """
-    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=1, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=false, enable_projection=true) */
+    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=1, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=false, enable_projection=true) */
         sum(l_extendedprice * l_discount) as revenue
     from
         lineitem
diff --git a/regression-test/suites/nereids_tpch_p0/tpch/q7.groovy b/regression-test/suites/nereids_tpch_p0/tpch/q7.groovy
index 75805d0911985f4..f490bc50316bd99 100644
--- a/regression-test/suites/nereids_tpch_p0/tpch/q7.groovy
+++ b/regression-test/suites/nereids_tpch_p0/tpch/q7.groovy
@@ -67,7 +67,7 @@ suite("nereids_tpch_q7") {
     """
 
     qt_select """
-    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=8, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=false, enable_projection=true) */
+    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=8, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=false, enable_projection=true) */
         supp_nation,
         cust_nation,
         l_year,
diff --git a/regression-test/suites/nereids_tpch_p0/tpch/q8.groovy b/regression-test/suites/nereids_tpch_p0/tpch/q8.groovy
index 444ad2c7d7ce381..068a3340000230c 100644
--- a/regression-test/suites/nereids_tpch_p0/tpch/q8.groovy
+++ b/regression-test/suites/nereids_tpch_p0/tpch/q8.groovy
@@ -66,7 +66,7 @@ order by
 
     qt_select """
 
-select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=8, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=false, enable_projection=true) */
+select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=8, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=false, enable_projection=true) */
     o_year,
     sum(case
         when nation = 'BRAZIL' then volume
diff --git a/regression-test/suites/nereids_tpch_p0/tpch/q9.groovy b/regression-test/suites/nereids_tpch_p0/tpch/q9.groovy
index 79dabaeefc9f964..efb47b3e1e20181 100644
--- a/regression-test/suites/nereids_tpch_p0/tpch/q9.groovy
+++ b/regression-test/suites/nereids_tpch_p0/tpch/q9.groovy
@@ -60,7 +60,7 @@ suite("nereids_tpch_q9") {
     """
 
     qt_select """
-    select/*+SET_VAR(exec_mem_limit=17179869184, parallel_fragment_exec_instance_num=4, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=false, enable_projection=true, enable_runtime_filter_prune=true, runtime_filter_wait_time_ms=10000) */
+    select/*+SET_VAR(exec_mem_limit=17179869184, parallel_pipeline_task_num=4, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=false, enable_projection=true, enable_runtime_filter_prune=true, runtime_filter_wait_time_ms=10000) */
         nation,
         o_year,
         sum(amount) as sum_profit
diff --git a/regression-test/suites/performance_p0/redundant_conjuncts.groovy b/regression-test/suites/performance_p0/redundant_conjuncts.groovy
index 6f9e223a5f222e9..3933fdd175cfb06 100644
--- a/regression-test/suites/performance_p0/redundant_conjuncts.groovy
+++ b/regression-test/suites/performance_p0/redundant_conjuncts.groovy
@@ -36,7 +36,7 @@ suite("redundant_conjuncts") {
 
     explain {
         sql("""
-        SELECT /*+SET_VAR(REWRITE_OR_TO_IN_PREDICATE_THRESHOLD=2, parallel_fragment_exec_instance_num = 1, enable_shared_scan = false) */ v1 FROM redundant_conjuncts WHERE k1 = 1 AND k1 = 1;
+        SELECT /*+SET_VAR(REWRITE_OR_TO_IN_PREDICATE_THRESHOLD=2, parallel_pipeline_task_num = 1, enable_shared_scan = false) */ v1 FROM redundant_conjuncts WHERE k1 = 1 AND k1 = 1;
         """)
         notContains "AND"
     }
diff --git a/regression-test/suites/query_p0/join/test_join_bucket_shffule_hash_partitioned.groovy b/regression-test/suites/query_p0/join/test_join_bucket_shffule_hash_partitioned.groovy
index 0af93839b1cdbca..c4ba62e72022edf 100644
--- a/regression-test/suites/query_p0/join/test_join_bucket_shffule_hash_partitioned.groovy
+++ b/regression-test/suites/query_p0/join/test_join_bucket_shffule_hash_partitioned.groovy
@@ -48,7 +48,7 @@ suite("test_join_bucket_shffule_hash_partitioned", "query,p0") {
                           (2,2,"2023-04-26 00:00:00", 2);
     """
 
-    qt_sql_join1 """ select /*+SET_VAR(parallel_fragment_exec_instance_num=1)*/
+    qt_sql_join1 """ select /*+SET_VAR(parallel_pipeline_task_num=1)*/
         t1.k1,
         t1.k2,
         t1.kdate,
@@ -69,7 +69,7 @@ suite("test_join_bucket_shffule_hash_partitioned", "query,p0") {
         t2.kdate;
     """
 
-     qt_sql_join1 """ select /*+SET_VAR(parallel_fragment_exec_instance_num=4)*/
+     qt_sql_join1 """ select /*+SET_VAR(parallel_pipeline_task_num=4)*/
         t1.k1,
         t1.k2,
         t1.kdate,
diff --git a/regression-test/suites/query_p0/join/test_join_result_count.groovy b/regression-test/suites/query_p0/join/test_join_result_count.groovy
index 775d9b37ddd485f..d14cb8c239341fa 100644
--- a/regression-test/suites/query_p0/join/test_join_result_count.groovy
+++ b/regression-test/suites/query_p0/join/test_join_result_count.groovy
@@ -151,7 +151,7 @@ INSERT INTO `t4` VALUES (NULL,'4M','DORIS','td','1','kQ','6G','XM','k3','Jp','02
 """
 
     sql """
-set parallel_fragment_exec_instance_num = 1;
+set parallel_pipeline_task_num = 1;
 """
     sql """
 set enable_pipeline_engine = 0;
diff --git a/regression-test/suites/query_p0/join/test_partitioned_hash_join.groovy b/regression-test/suites/query_p0/join/test_partitioned_hash_join.groovy
index 33a2595de2d9d9f..27a4f7db164deb0 100644
--- a/regression-test/suites/query_p0/join/test_partitioned_hash_join.groovy
+++ b/regression-test/suites/query_p0/join/test_partitioned_hash_join.groovy
@@ -65,7 +65,7 @@ suite("test_partitioned_hash_join", "query,p0,arrow_flight_sql") {
 
     qt_partitioned_hash_join1 """
         select
-            /*+SET_VAR(disable_join_reorder=true,experimental_enable_pipeline_engine=false, parallel_fragment_exec_instance_num=1)*/
+            /*+SET_VAR(disable_join_reorder=true,experimental_enable_pipeline_engine=false)*/
             kl1
         from
             test_partitioned_hash_join_l
diff --git a/regression-test/suites/query_p0/join/test_runtime_filter_decimal256.groovy b/regression-test/suites/query_p0/join/test_runtime_filter_decimal256.groovy
index d65bacc86621ec6..35a6fedb3559c3a 100644
--- a/regression-test/suites/query_p0/join/test_runtime_filter_decimal256.groovy
+++ b/regression-test/suites/query_p0/join/test_runtime_filter_decimal256.groovy
@@ -19,7 +19,7 @@ suite("test_runtime_filter_decimal256", "query_p0") {
     sql "set enable_runtime_filter_prune=false;"
     sql "set enable_nereids_planner = true;"
     sql "set enable_decimal256 = true;"
-    sql "set parallel_fragment_exec_instance_num = 4;"
+    sql "set parallel_pipeline_task_num = 4;"
 
     sql "drop table if exists test_runtime_filter_decimal256_0;"
     sql """ create table test_runtime_filter_decimal256_0(k1 int, v1 decimal(38, 6), v2 decimal(38, 6))
diff --git a/regression-test/suites/query_p0/session_variable/test_invalid_session.groovy b/regression-test/suites/query_p0/session_variable/test_invalid_session.groovy
index 198002847935f91..6f868fccac08714 100644
--- a/regression-test/suites/query_p0/session_variable/test_invalid_session.groovy
+++ b/regression-test/suites/query_p0/session_variable/test_invalid_session.groovy
@@ -21,10 +21,4 @@ suite("test_invalid_session") {
     } catch (Exception ex) {
         assert("${ex}".contains("parallel_pipeline_task_num value should greater than or equal 0, you set value is: -1"))
     }
-
-    try {
-        sql "set parallel_fragment_exec_instance_num = 0;"
-    } catch (Exception ex) {
-        assert("${ex}".contains("parallel_fragment_exec_instance_num value should greater than or equal 1, you set value is: 0"))
-    }
 }
diff --git a/regression-test/suites/query_p0/sort/sort.groovy b/regression-test/suites/query_p0/sort/sort.groovy
index 7742bdcd7664a3a..999e6743730acb1 100644
--- a/regression-test/suites/query_p0/sort/sort.groovy
+++ b/regression-test/suites/query_p0/sort/sort.groovy
@@ -152,9 +152,9 @@ suite("sort") {
     sb.append("""('2023-03-21 08:00:00', 1.1,1)""")
     sql """ ${sb.toString()} """
 
-    qt_order_by_float """ select /*SET_VAR(parallel_pipeline_task_num=1,parallel_fragment_exec_instance_num=1)*/ * from ${tblName} order by dc; """
-    qt_order_by_int """ select /*SET_VAR(parallel_pipeline_task_num=1,parallel_fragment_exec_instance_num=1)*/ * from ${tblName} order by ic; """
-    qt_order_by_uint """ select /*SET_VAR(parallel_pipeline_task_num=1,parallel_fragment_exec_instance_num=1)*/ * from ${tblName} order by time_period; """
+    qt_order_by_float """ select /*SET_VAR(parallel_pipeline_task_num=1)*/ * from ${tblName} order by dc; """
+    qt_order_by_int """ select /*SET_VAR(parallel_pipeline_task_num=1)*/ * from ${tblName} order by ic; """
+    qt_order_by_uint """ select /*SET_VAR(parallel_pipeline_task_num=1)*/ * from ${tblName} order by time_period; """
 
 
     // string order by test
diff --git a/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_retention.sql b/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_retention.sql
index 51fa7c677fe02d5..9451c73ab624f46 100644
--- a/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_retention.sql
+++ b/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_retention.sql
@@ -43,7 +43,7 @@ SELECT
             GROUP BY uid 
             ORDER BY uid ASC;
 
-SET parallel_fragment_exec_instance_num=4;
+SET parallel_pipeline_task_num=4;
 
 SELECT * from retention_test ORDER BY uid;
 
diff --git a/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy b/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy
index 523cfc0a0cd8a07..98e227da88c2595 100644
--- a/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy
+++ b/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy
@@ -625,8 +625,8 @@ suite("test_bitmap_function") {
     // qt_sql """ select orthogonal_bitmap_intersect_count(members, tag_group, 1150000, 1150001, 390006) from ${arthogonalBitmapTable} where  tag_group in ( 1150000, 1150001, 390006); """
     // qt_sql """ select orthogonal_bitmap_union_count(members) from ${arthogonalBitmapTable} where  tag_group in ( 1150000, 1150001, 390006);  """
     // qt_sql_orthogonal_bitmap_intersect_count2 """ select orthogonal_bitmap_intersect_count(members, tag_group, 1,2) from test_arthogonal_bitmap; """
-    // qt_sql_orthogonal_bitmap_intersect_count3_1 """ select /*+SET_VAR(parallel_fragment_exec_instance_num=1)*/orthogonal_bitmap_intersect_count(members, tag_group, 1,11) from test_arthogonal_bitmap; """
-    // qt_sql_orthogonal_bitmap_intersect_count3_2 """ select /*+SET_VAR(parallel_fragment_exec_instance_num=2)*/orthogonal_bitmap_intersect_count(members, tag_group, 1,11) from test_arthogonal_bitmap; """
+    // qt_sql_orthogonal_bitmap_intersect_count3_1 """ select /*+SET_VAR(parallel_pipeline_task_num=1)*/orthogonal_bitmap_intersect_count(members, tag_group, 1,11) from test_arthogonal_bitmap; """
+    // qt_sql_orthogonal_bitmap_intersect_count3_2 """ select /*+SET_VAR(parallel_pipeline_task_num=2)*/orthogonal_bitmap_intersect_count(members, tag_group, 1,11) from test_arthogonal_bitmap; """
     // qt_sql_orthogonal_bitmap_intersect_count4 """ select orthogonal_bitmap_intersect_count(members, tag_group, 2,12) from test_arthogonal_bitmap; """
     qt_sql_orthogonal_bitmap_union_count2 """ select orthogonal_bitmap_union_count( cast(null as bitmap)) from test_arthogonal_bitmap; """
     qt_sql_orthogonal_bitmap_union_count3 """ select orthogonal_bitmap_union_count(members) from test_arthogonal_bitmap; """
@@ -692,7 +692,7 @@ suite("test_bitmap_function") {
             (17, to_bitmap(44));
     """
     qt_sql_bitmap_subset_in_range """
-        select /*+SET_VAR(parallel_fragment_exec_instance_num=1)*/
+        select /*+SET_VAR(parallel_pipeline_task_num=1)*/
             bitmap_to_string(
                 bitmap_subset_in_range(id, cast(null as bigint), cast(null as bigint))
             )
diff --git a/regression-test/suites/query_p0/sql_functions/conditional_functions/test_nullif.groovy b/regression-test/suites/query_p0/sql_functions/conditional_functions/test_nullif.groovy
index 1cbe69ed913dc9f..a644319c5225b2c 100644
--- a/regression-test/suites/query_p0/sql_functions/conditional_functions/test_nullif.groovy
+++ b/regression-test/suites/query_p0/sql_functions/conditional_functions/test_nullif.groovy
@@ -97,11 +97,11 @@ suite("test_nullif") {
     qt_if_nullif8 """select ifnull(b.k1, -1) k1 from baseall a left join bigtable b on a.k1 = b.k1 + 5 
             order by a.k1"""
     // make sure stable
-    qt_if_nullif8_1 """select   /*+ SET_VAR(enable_pipeline_engine=false,parallel_fragment_exec_instance_num=2,enable_share_hash_table_for_broadcast_join=true) */ b.k1, ifnull(b.k1, -1) k1 from baseall a left join bigtable b on a.k1 = b.k1 + 5 order by a.k1;"""
-    qt_if_nullif8_2 """select   /*+ SET_VAR(enable_pipeline_engine=false,parallel_fragment_exec_instance_num=2,enable_share_hash_table_for_broadcast_join=true) */ b.k1, ifnull(b.k1, -1) k1 from baseall a left join bigtable b on a.k1 = b.k1 + 5 order by a.k1;"""
-    qt_if_nullif8_3 """select   /*+ SET_VAR(enable_pipeline_engine=false,parallel_fragment_exec_instance_num=2,enable_share_hash_table_for_broadcast_join=true) */ b.k1, ifnull(b.k1, -1) k1 from baseall a left join bigtable b on a.k1 = b.k1 + 5 order by a.k1;"""
-    qt_if_nullif8_4 """select   /*+ SET_VAR(enable_pipeline_engine=false,parallel_fragment_exec_instance_num=2,enable_share_hash_table_for_broadcast_join=true) */ b.k1, ifnull(b.k1, -1) k1 from baseall a left join bigtable b on a.k1 = b.k1 + 5 order by a.k1;"""
-    qt_if_nullif8_5 """select   /*+ SET_VAR(enable_pipeline_engine=false,parallel_fragment_exec_instance_num=2,enable_share_hash_table_for_broadcast_join=true) */ b.k1, ifnull(b.k1, -1) k1 from baseall a left join bigtable b on a.k1 = b.k1 + 5 order by a.k1;"""
+    qt_if_nullif8_1 """select   /*+ SET_VAR(enable_pipeline_engine=false,parallel_pipeline_task_num=2,enable_share_hash_table_for_broadcast_join=true) */ b.k1, ifnull(b.k1, -1) k1 from baseall a left join bigtable b on a.k1 = b.k1 + 5 order by a.k1;"""
+    qt_if_nullif8_2 """select   /*+ SET_VAR(enable_pipeline_engine=false,parallel_pipeline_task_num=2,enable_share_hash_table_for_broadcast_join=true) */ b.k1, ifnull(b.k1, -1) k1 from baseall a left join bigtable b on a.k1 = b.k1 + 5 order by a.k1;"""
+    qt_if_nullif8_3 """select   /*+ SET_VAR(enable_pipeline_engine=false,parallel_pipeline_task_num=2,enable_share_hash_table_for_broadcast_join=true) */ b.k1, ifnull(b.k1, -1) k1 from baseall a left join bigtable b on a.k1 = b.k1 + 5 order by a.k1;"""
+    qt_if_nullif8_4 """select   /*+ SET_VAR(enable_pipeline_engine=false,parallel_pipeline_task_num=2,enable_share_hash_table_for_broadcast_join=true) */ b.k1, ifnull(b.k1, -1) k1 from baseall a left join bigtable b on a.k1 = b.k1 + 5 order by a.k1;"""
+    qt_if_nullif8_5 """select   /*+ SET_VAR(enable_pipeline_engine=false,parallel_pipeline_task_num=2,enable_share_hash_table_for_broadcast_join=true) */ b.k1, ifnull(b.k1, -1) k1 from baseall a left join bigtable b on a.k1 = b.k1 + 5 order by a.k1;"""
     qt_if_nullif10 """select ifnull(b.k6, "hll") k1 from baseall a left join bigtable b on a.k1 = b.k1 + 5 
             order by k1"""
     qt_if_nullif11 """select ifnull(b.k10, "2017-06-06") k1 from baseall a left join bigtable b on 
diff --git a/regression-test/suites/query_p0/sql_functions/conditional_functions/test_query_limit.groovy b/regression-test/suites/query_p0/sql_functions/conditional_functions/test_query_limit.groovy
index 87becfe020dd8d2..b442c45897eb70d 100644
--- a/regression-test/suites/query_p0/sql_functions/conditional_functions/test_query_limit.groovy
+++ b/regression-test/suites/query_p0/sql_functions/conditional_functions/test_query_limit.groovy
@@ -73,15 +73,15 @@ suite("test_query_limit", "query,p0") {
     qt_limit16 "select * from (select * from ${tableName} order by k1, k2, k3, k4 limit 1, 2) a limit 2, 2"
     qt_limit17 "select * from (select * from ${tableName} order by k1, k2, k3, k4 limit 1, 2) a limit 2, 3"
     test {
-        sql "select /*+SET_VAR(parallel_fragment_exec_instance_num=4)*/ * from ${tableName} limit 1, 10"
+        sql "select /*+SET_VAR(parallel_pipeline_task_num=4)*/ * from ${tableName} limit 1, 10"
         rowNum 2
     }
     test {
-        sql "select /*+SET_VAR(parallel_fragment_exec_instance_num=4)*/ * from ${tableName} limit 2, 10"
+        sql "select /*+SET_VAR(parallel_pipeline_task_num=4)*/ * from ${tableName} limit 2, 10"
         rowNum 1
     }
     test {
-        sql "select /*+SET_VAR(parallel_fragment_exec_instance_num=4)*/ * from ${tableName} limit 3, 10"
+        sql "select /*+SET_VAR(parallel_pipeline_task_num=4)*/ * from ${tableName} limit 3, 10"
         rowNum 0
     }
 }
diff --git a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy
index 8bd43dae685afcb..50823de77c8b8a1 100644
--- a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy
+++ b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy
@@ -77,7 +77,7 @@ suite("test_date_function") {
                 ("2019-08-01 13:21:03"),
                 ("2019-08-01 13:21:03");
     """
-    qt_sql_convert_tz_null """ SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1)*/ convert_tz(test_datetime, cast(null as varchar), cast(null as varchar)) result from test_date_function; """
+    qt_sql_convert_tz_null """ SELECT /*+SET_VAR(parallel_pipeline_task_num=1)*/ convert_tz(test_datetime, cast(null as varchar), cast(null as varchar)) result from test_date_function; """
 
     sql """ truncate table ${tableName} """
 
@@ -122,7 +122,7 @@ suite("test_date_function") {
             (18, "2019-08-08 13:21:03", "Africa/Lusaka", "America/Creston")
     """
 
-    sql "set parallel_fragment_exec_instance_num = 8"
+    sql "set parallel_pipeline_task_num = 8"
 
     qt_sql1 """
         SELECT
diff --git a/regression-test/suites/query_p0/sql_functions/math_functions/test_conv.groovy b/regression-test/suites/query_p0/sql_functions/math_functions/test_conv.groovy
index 3a74abfe9c8b221..68a18f8f0760bdf 100644
--- a/regression-test/suites/query_p0/sql_functions/math_functions/test_conv.groovy
+++ b/regression-test/suites/query_p0/sql_functions/math_functions/test_conv.groovy
@@ -45,6 +45,6 @@ suite("test_conv", "arrow_flight_sql") {
         ("100", 1)
     """
 
-    qt_sql_conv1 """ select /*+SET_VAR(parallel_fragment_exec_instance_num=1)*/conv(k1, cast(null as bigint), cast(null as bigint)) from test_conv; """
+    qt_sql_conv1 """ select /*+SET_VAR(parallel_pipeline_task_num=1)*/conv(k1, cast(null as bigint), cast(null as bigint)) from test_conv; """
 }
 
diff --git a/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function.groovy b/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function.groovy
index c1a3e6b57fe8098..9a44bf6d75691d7 100644
--- a/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function.groovy
+++ b/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function.groovy
@@ -268,7 +268,7 @@ suite("test_string_function", "arrow_flight_sql") {
         ("aaaaaaaa", 1)
     """
     // bug fix
-    qt_sql_substring1 """ select /*+SET_VAR(parallel_fragment_exec_instance_num=1)*/ substring(k1, cast(null as int), cast(null as int)) from test_string_function; """
+    qt_sql_substring1 """ select /*+SET_VAR(parallel_pipeline_task_num=1)*/ substring(k1, cast(null as int), cast(null as int)) from test_string_function; """
 
     qt_sql "select substr('a',3,1);"
     qt_sql "select substr('a',2,1);"
diff --git a/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function_regexp.groovy b/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function_regexp.groovy
index 2066452b0d6c4e0..546ebb7fd100aaf 100644
--- a/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function_regexp.groovy
+++ b/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function_regexp.groovy
@@ -91,7 +91,7 @@ suite("test_string_function_regexp") {
             ("billie eillish"),
             ("billie eillish")
         """
-    qt_sql_regexp_null "SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1)*/regexp_extract(k, cast(null as varchar), 1) from test_string_function_regexp;"
+    qt_sql_regexp_null "SELECT /*+SET_VAR(parallel_pipeline_task_num=1)*/regexp_extract(k, cast(null as varchar), 1) from test_string_function_regexp;"
     // end bug fix
 
     sql "DROP TABLE ${tbName};"
diff --git a/regression-test/suites/query_p0/sql_functions/window_functions/test_window_function.groovy b/regression-test/suites/query_p0/sql_functions/window_functions/test_window_function.groovy
index 7354625c1cd6efb..c27a6bdbbdd3e7f 100644
--- a/regression-test/suites/query_p0/sql_functions/window_functions/test_window_function.groovy
+++ b/regression-test/suites/query_p0/sql_functions/window_functions/test_window_function.groovy
@@ -23,7 +23,7 @@ suite("test_window_function") {
     sql """ INSERT INTO ${windowFunctionTable1} VALUES ('JDR',12.86,'2014-10-02 00:00:00','2014-10-02 00:00:00.111111','2014-10-02 00:00:00.111111','2014-10-02 00:00:00.111111'),('JDR',12.89,'2014-10-03 00:00:00','2014-10-03 00:00:00.111111','2014-10-03 00:00:00.111111','2014-10-03 00:00:00.111111'),('JDR',12.94,'2014-10-04 00:00:00','2014-10-04 00:00:00.111111','2014-10-04 00:00:00.111111','2014-10-04 00:00:00.111111'),('JDR',12.55,'2014-10-05 00:00:00','2014-10-05 00:00:00.111111','2014-10-05 00:00:00.111111','2014-10-05 00:00:00.111111'),('JDR',14.03,'2014-10-06 00:00:00','2014-10-06 00:00:00.111111','2014-10-06 00:00:00.111111','2014-10-06 00:00:00.111111'),('JDR',14.75,'2014-10-07 00:00:00','2014-10-07 00:00:00.111111','2014-10-07 00:00:00.111111','2014-10-07 00:00:00.111111'),('JDR',13.98,'2014-10-08 00:00:00','2014-10-08 00:00:00.111111','2014-10-08 00:00:00.111111','2014-10-08 00:00:00.111111') """
 
     qt_sql """
-            SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+            SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                 stock_symbol,
                 closing_date,
                 closing_price,
@@ -37,7 +37,7 @@ suite("test_window_function") {
            """
     // LEAD
     qt_sql """ 
-             SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+             SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                 stock_symbol,
                 closing_date,
                 closing_price,
@@ -52,12 +52,12 @@ suite("test_window_function") {
 
     // LEAD not nullable coredump
     qt_sql """
-           select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ t1.new_time from (select closing_date, lead(closing_date, 1, '2014-10-02 00:00:00') over () as new_time from ${windowFunctionTable1}) as t1 left join ${windowFunctionTable1} t2 on t2.closing_date = t1.closing_date order by t1.new_time desc;
+           select /*+SET_VAR(parallel_pipeline_task_num=1) */ t1.new_time from (select closing_date, lead(closing_date, 1, '2014-10-02 00:00:00') over () as new_time from ${windowFunctionTable1}) as t1 left join ${windowFunctionTable1} t2 on t2.closing_date = t1.closing_date order by t1.new_time desc;
            """
 
     // LAG
     qt_sql """ 
-             SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+             SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                 stock_symbol,
                 closing_date,
                 closing_price,
@@ -69,7 +69,7 @@ suite("test_window_function") {
             """
 
     qt_sql """
-            SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+            SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                 stock_symbol,
                 closing_date1,
                 closing_price,
@@ -83,7 +83,7 @@ suite("test_window_function") {
            """
     // LEAD
     qt_sql """
-             SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+             SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                 stock_symbol,
                 closing_date1,
                 closing_price,
@@ -98,12 +98,12 @@ suite("test_window_function") {
 
     // LEAD not nullable coredump
     qt_sql """  
-           select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ t1.new_time from (select closing_date1, lead(closing_date1, 1, '2014-10-02 00:00:00') over () as new_time from ${windowFunctionTable1}) as t1 left join ${windowFunctionTable1} t2 on t2.closing_date1 = t1.closing_date1 order by t1.new_time desc;
+           select /*+SET_VAR(parallel_pipeline_task_num=1) */ t1.new_time from (select closing_date1, lead(closing_date1, 1, '2014-10-02 00:00:00') over () as new_time from ${windowFunctionTable1}) as t1 left join ${windowFunctionTable1} t2 on t2.closing_date1 = t1.closing_date1 order by t1.new_time desc;
            """
 
     // LAG
     qt_sql """
-             SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+             SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                 stock_symbol,
                 closing_date1,
                 closing_price,
@@ -115,7 +115,7 @@ suite("test_window_function") {
             """
 
     qt_sql """
-            SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+            SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                 stock_symbol,
                 closing_date2,
                 closing_price,
@@ -129,7 +129,7 @@ suite("test_window_function") {
            """
     // LEAD
     qt_sql """
-             SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+             SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                 stock_symbol,
                 closing_date2,
                 closing_price,
@@ -144,12 +144,12 @@ suite("test_window_function") {
 
     // LEAD not nullable coredump
     qt_sql """
-           select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ t1.new_time from (select closing_date2, lead(closing_date2, 1, '2014-10-02 00:00:00') over () as new_time from ${windowFunctionTable1}) as t1 left join ${windowFunctionTable1} t2 on t2.closing_date2 = t1.closing_date2 order by t1.new_time desc;
+           select /*+SET_VAR(parallel_pipeline_task_num=1) */ t1.new_time from (select closing_date2, lead(closing_date2, 1, '2014-10-02 00:00:00') over () as new_time from ${windowFunctionTable1}) as t1 left join ${windowFunctionTable1} t2 on t2.closing_date2 = t1.closing_date2 order by t1.new_time desc;
            """
 
     // LAG
     qt_sql """
-             SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+             SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                 stock_symbol,
                 closing_date2,
                 closing_price,
@@ -161,7 +161,7 @@ suite("test_window_function") {
             """
 
     qt_sql """
-            SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+            SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                 stock_symbol,
                 closing_date3,
                 closing_price,
@@ -175,7 +175,7 @@ suite("test_window_function") {
            """
     // LEAD
     qt_sql """
-             SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+             SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                 stock_symbol,
                 closing_date3,
                 closing_price,
@@ -190,12 +190,12 @@ suite("test_window_function") {
 
     // LEAD not nullable coredump
     qt_sql """
-           select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ t1.new_time from (select closing_date3, lead(closing_date3, 1, '2014-10-02 00:00:00') over () as new_time from ${windowFunctionTable1}) as t1 left join ${windowFunctionTable1} t2 on t2.closing_date3 = t1.closing_date3 order by t1.new_time desc;
+           select /*+SET_VAR(parallel_pipeline_task_num=1) */ t1.new_time from (select closing_date3, lead(closing_date3, 1, '2014-10-02 00:00:00') over () as new_time from ${windowFunctionTable1}) as t1 left join ${windowFunctionTable1} t2 on t2.closing_date3 = t1.closing_date3 order by t1.new_time desc;
            """
 
     // LAG
     qt_sql """
-             SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+             SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                 stock_symbol,
                 closing_date3,
                 closing_price,
@@ -215,7 +215,7 @@ suite("test_window_function") {
 
     // SUM
     qt_sql """
-               SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+               SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                     x,
                     property,
                     sum( x ) over ( PARTITION BY property ORDER BY x rows BETWEEN 1 preceding AND 1 following ) AS 'moving total' 
@@ -228,7 +228,7 @@ suite("test_window_function") {
            """
     // AVG
     qt_sql """
-               SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+               SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                     x,
                     property,
                     avg( x ) over ( PARTITION BY property ORDER BY x rows BETWEEN 1 preceding AND 1 following ) AS 'moving average' 
@@ -241,7 +241,7 @@ suite("test_window_function") {
            """
     // COUNT
     qt_sql """
-               SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+               SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                     x,
                     property,
                     count( x ) over ( PARTITION BY property ORDER BY x rows BETWEEN unbounded preceding AND current ROW ) AS 'cumulative total' 
@@ -257,7 +257,7 @@ suite("test_window_function") {
 
     // MIN
     qt_sql """ 
-              SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+              SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                    x,
                    property,
                    min( x ) over ( ORDER BY property, x DESC rows BETWEEN unbounded preceding AND 1 following ) AS 'local minimum' 
@@ -268,7 +268,7 @@ suite("test_window_function") {
            """
     // MAX
     qt_sql """
-              SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */
+              SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */
                    x,
                    property,
                    max( x ) over ( ORDER BY property, x rows BETWEEN unbounded preceding AND 1 following ) AS 'local maximum' 
@@ -286,15 +286,15 @@ suite("test_window_function") {
     sql """ insert into  ${windowFunctionTable3} values (1,1),(1,2),(1,2),(2,1),(2,2),(2,3),(3,1),(3,1),(3,2); """
 
     // RANK
-    qt_sql """ select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ x, y, rank() over(partition by x order by y) as rank from ${windowFunctionTable3} order by x, y; """
+    qt_sql """ select /*+SET_VAR(parallel_pipeline_task_num=1) */ x, y, rank() over(partition by x order by y) as rank from ${windowFunctionTable3} order by x, y; """
     // DENSE_RANK
-    qt_sql """ select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ x, y, dense_rank() over(partition by x order by y) as rank from ${windowFunctionTable3} order by x, y; """
+    qt_sql """ select /*+SET_VAR(parallel_pipeline_task_num=1) */ x, y, dense_rank() over(partition by x order by y) as rank from ${windowFunctionTable3} order by x, y; """
     // PERCENT_RANK
-    qt_sql """ select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ x, y, percent_rank() over(partition by x order by y) as rank from ${windowFunctionTable3} order by x, y; """
+    qt_sql """ select /*+SET_VAR(parallel_pipeline_task_num=1) */ x, y, percent_rank() over(partition by x order by y) as rank from ${windowFunctionTable3} order by x, y; """
     // CUME_DIST
-    qt_sql """ select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ x, y, cume_dist() over(partition by x order by y) as rank from ${windowFunctionTable3} order by x, y; """
+    qt_sql """ select /*+SET_VAR(parallel_pipeline_task_num=1) */ x, y, cume_dist() over(partition by x order by y) as rank from ${windowFunctionTable3} order by x, y; """
     // ROW_NUMBER
-    qt_sql """ select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ x, y, row_number() over(partition by x order by y) as rank from ${windowFunctionTable3} order by x, y; """
+    qt_sql """ select /*+SET_VAR(parallel_pipeline_task_num=1) */ x, y, row_number() over(partition by x order by y) as rank from ${windowFunctionTable3} order by x, y; """
 
     sql """ drop table   ${windowFunctionTable3}  """
 
@@ -305,9 +305,9 @@ suite("test_window_function") {
     sql """ insert into ${windowFunctionTable4} VALUES ('Pete','USA','Hello'),('John','USA','Hi'),('Boris','Germany','Guten tag'),('Michael','Germany','Guten morgen'),('Bjorn','Sweden','Hej'),('Mats','Sweden','Tja')"""
 
     // first_value
-    qt_sql """ select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ country, name,first_value(greeting) over (partition by country order by name, greeting) as greeting from ${windowFunctionTable4} order by country, name; """
+    qt_sql """ select /*+SET_VAR(parallel_pipeline_task_num=1) */ country, name,first_value(greeting) over (partition by country order by name, greeting) as greeting from ${windowFunctionTable4} order by country, name; """
     // last_value
-    qt_sql """ select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ country, name,last_value(greeting)  over (partition by country order by name, greeting) as greeting from ${windowFunctionTable4} order by country, name; """
+    qt_sql """ select /*+SET_VAR(parallel_pipeline_task_num=1) */ country, name,last_value(greeting)  over (partition by country order by name, greeting) as greeting from ${windowFunctionTable4} order by country, name; """
 
     sql """ drop table   ${windowFunctionTable4}  """
 
@@ -318,15 +318,15 @@ suite("test_window_function") {
     String k1 = fields[3]
     String k2 = fields[5]
     String k3 = fields[3]
-    qt_first_value1"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, first_value(${k2}) over (partition by ${k1} order by ${k3}, ${k2})
+    qt_first_value1"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, first_value(${k2}) over (partition by ${k1} order by ${k3}, ${k2})
              as wj from baseall  order by ${k1}, wj"""
-    qt_first_value2"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, first_value(${k2}) over (partition by ${k1} order by ${k3}, ${k2}
+    qt_first_value2"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, first_value(${k2}) over (partition by ${k1} order by ${k3}, ${k2}
              range between unbounded preceding and current row)
              as wj from baseall  order by ${k1}, wj"""
-    qt_first_value3"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, first_value(${k2}) over (partition by ${k1} order by ${k3}, ${k2} 
+    qt_first_value3"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, first_value(${k2}) over (partition by ${k1} order by ${k3}, ${k2}
              rows between unbounded preceding and current row)
              as wj from baseall  order by ${k1}, wj"""
-    qt_first_value4"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ a, min(d) as wjj from 
+    qt_first_value4"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ a, min(d) as wjj from
              (select t1.k1 as k1, t1.k2 as k2, t1.k3 as k3,
              t1.k4 as k4, t1.k5 as k5,t1.k6 as k6,
              t1.k7 as k7, t1.k8 as k8, t1.k9 as k9,
@@ -338,15 +338,15 @@ suite("test_window_function") {
              order by a, wjj"""
 
     // test_query_last_value
-    qt_last_value1"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, last_value(${k2}) over (partition by ${k1} order by ${k3},${k2})
+    qt_last_value1"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, last_value(${k2}) over (partition by ${k1} order by ${k3},${k2})
                  as wj from baseall  order by ${k1}, wj"""
-    qt_last_value2"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, last_value(${k2}) over (partition by ${k1} order by ${k3},${k2}
+    qt_last_value2"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, last_value(${k2}) over (partition by ${k1} order by ${k3},${k2}
              range between unbounded preceding and current row)
              as wj from baseall  order by ${k1}, wj"""
-    qt_last_value3"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, last_value(${k2}) over (partition by ${k1} order by ${k3},${k2}
+    qt_last_value3"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, last_value(${k2}) over (partition by ${k1} order by ${k3},${k2}
              rows between unbounded preceding and current row)
              as wj from baseall  order by ${k1}, wj"""
-    qt_last_value4"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ a, max(d) as wjj from 
+    qt_last_value4"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ a, max(d) as wjj from
              (select t1.k1 as k1, t1.k2 as k2, t1.k3 as k3,
              t1.k4 as k4, t1.k5 as k5,t1.k6 as k6,
              t1.k7 as k7, t1.k8 as k8, t1.k9 as k9,
@@ -358,79 +358,79 @@ suite("test_window_function") {
              order by a, wjj"""
 
     // test_query_row_number
-    qt_row_number1"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, row_number() over (partition by ${k1} order by ${k3}) 
+    qt_row_number1"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, row_number() over (partition by ${k1} order by ${k3})
                     as wj from baseall order by ${k1}, wj"""
-    qt_row_number2"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, count(k1) over (partition by ${k1} order by ${k3}
+    qt_row_number2"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, count(k1) over (partition by ${k1} order by ${k3}
                     rows between unbounded preceding and current row)
                     as wj from baseall order by ${k1}, wj"""
 
     // test error
     test {
-        sql("select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, lag(${k2}) over (partition by ${k1} order by ${k3}) from baseall")
+        sql("select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, lag(${k2}) over (partition by ${k1} order by ${k3}) from baseall")
         exception ""
     }
     test {
-        sql"select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, lag(${k2}, -1, 1) over (partition by ${k1} order by ${k3}) from baseall"
+        sql"select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, lag(${k2}, -1, 1) over (partition by ${k1} order by ${k3}) from baseall"
         exception ""
     }
     test {
-        sql"select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, lag(${k2}, 1) over (partition by ${k1} order by ${k3}) from baseall"
+        sql"select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, lag(${k2}, 1) over (partition by ${k1} order by ${k3}) from baseall"
         exception ""
     }
     test {
-        sql"select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, lead(${k2}) over (partition by ${k1} order by ${k3}) from baseall"
+        sql"select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, lead(${k2}) over (partition by ${k1} order by ${k3}) from baseall"
         exception ""
     }
     test {
-        sql"select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, lead(${k2}, -1, 1) over (partition by ${k1} order by ${k3}) from baseall"
+        sql"select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, lead(${k2}, -1, 1) over (partition by ${k1} order by ${k3}) from baseall"
         exception ""
     }
     test {
-        sql"select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, lead(${k2}, 1) over (partition by ${k1} order by ${k3}) from baseall"
+        sql"select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, lead(${k2}, 1) over (partition by ${k1} order by ${k3}) from baseall"
         exception ""
     }
-    qt_window_error1"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, first_value(${k2}) over (partition by ${k1}) from baseall order by ${k1}"""
-    qt_window_error2"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, first_value(${k2}) over (order by ${k3}) from baseall"""
-    qt_window_error3"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, max(${k2}) over (order by ${k3}) from baseall"""
+    qt_window_error1"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, first_value(${k2}) over (partition by ${k1}) from baseall order by ${k1}"""
+    qt_window_error2"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, first_value(${k2}) over (order by ${k3}) from baseall"""
+    qt_window_error3"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, max(${k2}) over (order by ${k3}) from baseall"""
     test {
-        sql"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, sum(${k2}) over (partition by ${k1} order by ${k3} rows
+        sql"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, sum(${k2}) over (partition by ${k1} order by ${k3} rows
             between current row and unbounded preceding) as wj
             from baseall order by ${k1}, wj"""
         exception ""
     }
     test {
-        sql"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, sum(${k2}) over (partition by ${k1} order by ${k3} rows 
+        sql"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, sum(${k2}) over (partition by ${k1} order by ${k3} rows
             between 0 preceding and 1 following) as wj 
             from baseall order by ${k1}, wj"""
         exception ""
     }
     test {
-        sql"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, sum(${k2}) over (partition by ${k1} order by ${k3} rows 
+        sql"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, sum(${k2}) over (partition by ${k1} order by ${k3} rows
             between unbounded following and current row) as wj 
             from baseall order by ${k1}, wj"""
         exception ""
     }
     test {
-        sql"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, rank(${k2}) over (partition by ${k1} order by ${k3}) as wj
+        sql"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, rank(${k2}) over (partition by ${k1} order by ${k3}) as wj
             from baseall order by ${k1}, wj"""
         exception ""
     }
     test {
-        sql"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, max() over (partition by ${k1} order by ${k3}) as wj
+        sql"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, max() over (partition by ${k1} order by ${k3}) as wj
             from baseall order by ${k1}, wj"""
         exception ""
     }
     test {
-        sql"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, count(${k2}) over (order by ${k1} rows partition by ${k3}) as wj
+        sql"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, count(${k2}) over (order by ${k1} rows partition by ${k3}) as wj
             from baseall order by ${k1}, wj"""
         exception ""
     }
 
     // test_query_rank
     k3 = fields[7]
-    qt_rank1"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, rank() over (partition by ${k1} order by ${k3}) as wj 
+    qt_rank1"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, rank() over (partition by ${k1} order by ${k3}) as wj
              from baseall order by ${k1}, wj"""
-    qt_rank2"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ F2.${k1}, (F1.wj - F2.basewj + 1) as wj from
+    qt_rank2"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ F2.${k1}, (F1.wj - F2.basewj + 1) as wj from
              (select a, c, count(*) as wj from 
              (select t1.k1 as k1, t1.k2 as k2, t1.k3 as k3,
              t1.k4 as k4, t1.k5 as k5,t1.k6 as k6,
@@ -444,16 +444,16 @@ suite("test_window_function") {
              where F1.a=F2.${k1} and F1.c = F2.${k3} order by F2.${k1}, wj"""
 
     //test_hang
-    qt_window_hang1"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, row_number() over (partition by ${k1} order by ${k3}) as wj from
+    qt_window_hang1"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, row_number() over (partition by ${k1} order by ${k3}) as wj from
              baseall order by ${k1}, wj"""
     String line = "("
     String cur
     for (p in range(0, 829)) {
         if (p == 0) {
-            cur = "(select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, 1 as wj from baseall order by ${k1}, ${k3} limit 1)".toString()
+            cur = "(select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, 1 as wj from baseall order by ${k1}, ${k3} limit 1)".toString()
         }
         else {
-            cur = """(select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ ${k1}, ${p+1} as wj from baseall order by ${k1} , ${k3}
+            cur = """(select /*+SET_VAR(parallel_pipeline_task_num=1) */ ${k1}, ${p+1} as wj from baseall order by ${k1} , ${k3}
                     limit ${p}, 1 ) """.toString()
 
         }
@@ -467,7 +467,7 @@ suite("test_window_function") {
 
     sql """ admin set frontend config("remote_fragment_exec_timeout_ms"="300000"); """
 
-    //qt_window_hang2"""select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ A.${k1}, A.wj - B.dyk + 1 as num from 
+    //qt_window_hang2"""select /*+SET_VAR(parallel_pipeline_task_num=1) */ A.${k1}, A.wj - B.dyk + 1 as num from
     //    (select ${k1}, wj from ${line} as W1) as A join 
     //    (select ${k1}, min(wj) as dyk from ${line} as W2 group by ${k1}) as B
     //    where A.${k1}=B.${k1}  order by A.${k1}, num"""
@@ -476,9 +476,9 @@ suite("test_window_function") {
     line = "("
     for (p in range(0, 829)) {
         if (p == 0 ) {
-            cur = "(select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ * from baseall order by k1, k6 limit 1)"
+            cur = "(select /*+SET_VAR(parallel_pipeline_task_num=1) */ * from baseall order by k1, k6 limit 1)"
         } else {
-            cur = "(select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ * from baseall order by k1, k6 limit ${p}, 1)"
+            cur = "(select /*+SET_VAR(parallel_pipeline_task_num=1) */ * from baseall order by k1, k6 limit ${p}, 1)"
         }
         if (p < 828) {
             line = line + cur + " union all "
@@ -487,12 +487,12 @@ suite("test_window_function") {
         }
     }
     // qt_hujie1"select T.k1, T.k6 from ${line} as T order by T.k1, T.k6"
-    qt_hujie2"select /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ k1, k6 from baseall order by k1, k6"
+    qt_hujie2"select /*+SET_VAR(parallel_pipeline_task_num=1) */ k1, k6 from baseall order by k1, k6"
 
     // test_bug
-    order_qt_window_bug1"""SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ wj FROM (SELECT row_number() over (PARTITION BY k6 ORDER BY k1) AS wj 
+    order_qt_window_bug1"""SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */ wj FROM (SELECT row_number() over (PARTITION BY k6 ORDER BY k1) AS wj
             FROM baseall ) AS A where wj = 2"""
-    order_qt_window_bug2"""SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1) */ A.k2 AS a, A.k1 as b, B.k1 as c, B.k2 as d FROM  
+    order_qt_window_bug2"""SELECT /*+SET_VAR(parallel_pipeline_task_num=1) */ A.k2 AS a, A.k1 as b, B.k1 as c, B.k2 as d FROM
            ( SELECT k2, k1, row_number () over (PARTITION BY k2 ORDER BY k3) AS wj  
            FROM baseall ) AS A JOIN ( SELECT k2, k1, row_number () over  
            (PARTITION BY k2 ORDER BY k3) AS wj FROM baseall ) AS B WHERE A.k2=B.k2"""
diff --git a/regression-test/suites/shape_check/tpcds_sf100/ddl/rf_prune.tmpl b/regression-test/suites/shape_check/tpcds_sf100/ddl/rf_prune.tmpl
index 36ee85ca202f675..9a9c2f2380d3fc1 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/ddl/rf_prune.tmpl
+++ b/regression-test/suites/shape_check/tpcds_sf100/ddl/rf_prune.tmpl
@@ -28,7 +28,6 @@ suite("query{--}") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/ddl/shape.tmpl b/regression-test/suites/shape_check/tpcds_sf100/ddl/shape.tmpl
index bd8228ffd92b699..a6ace12c4b26a89 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/ddl/shape.tmpl
+++ b/regression-test/suites/shape_check/tpcds_sf100/ddl/shape.tmpl
@@ -28,7 +28,6 @@ suite("query{--}") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query1.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query1.groovy
index 58da7df182f9333..5fd3d6072a45001 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query1.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query1.groovy
@@ -28,7 +28,6 @@ suite("query1") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query10.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query10.groovy
index 46186ff15743b4f..0142e9b010d036c 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query10.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query10.groovy
@@ -28,7 +28,6 @@ suite("query10") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query11.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query11.groovy
index 4c36de065b39d91..b0f042e0eb987e8 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query11.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query11.groovy
@@ -28,7 +28,6 @@ suite("query11") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query12.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query12.groovy
index 8c45841c8b7b09b..2b122912686837c 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query12.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query12.groovy
@@ -28,7 +28,6 @@ suite("query12") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query13.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query13.groovy
index 432b39a4728c25f..42e398817b86243 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query13.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query13.groovy
@@ -28,7 +28,6 @@ suite("query13") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query14.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query14.groovy
index 95bab9c6e316090..a731a8c6bf1d82f 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query14.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query14.groovy
@@ -28,7 +28,6 @@ suite("query14") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query15.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query15.groovy
index 2e572df7dc89200..cd4d523e6b7e651 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query15.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query15.groovy
@@ -28,7 +28,6 @@ suite("query15") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query16.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query16.groovy
index 359c696f1d483d6..9fb3ed3474f647a 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query16.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query16.groovy
@@ -28,7 +28,6 @@ suite("query16") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query17.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query17.groovy
index 70cfcb6cd32c53e..fe304307a8462e4 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query17.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query17.groovy
@@ -28,7 +28,6 @@ suite("query17") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query18.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query18.groovy
index bc1c5804dd5b012..d710df78b982a4a 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query18.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query18.groovy
@@ -28,7 +28,6 @@ suite("query18") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query19.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query19.groovy
index d92ddd75803531f..cf153826c0cc3d4 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query19.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query19.groovy
@@ -28,7 +28,6 @@ suite("query19") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query2.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query2.groovy
index 069a3b70e0119d7..f22a96caf127446 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query2.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query2.groovy
@@ -28,7 +28,6 @@ suite("query2") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query20.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query20.groovy
index 8d3e3d2650bec97..52379aef5cae9af 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query20.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query20.groovy
@@ -28,7 +28,6 @@ suite("query20") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query21.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query21.groovy
index 250a8c7499b3742..8937f0cbe14de18 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query21.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query21.groovy
@@ -29,7 +29,6 @@ suite("query21") {
     sql 'SET enable_fold_constant_by_be = false' //plan shape will be different
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query22.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query22.groovy
index c5bc45019adbb3f..ee7d6715d0c6121 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query22.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query22.groovy
@@ -28,7 +28,6 @@ suite("query22") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query24.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query24.groovy
index 1107cb92bc9547a..387a9c0ce028356 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query24.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query24.groovy
@@ -28,7 +28,6 @@ suite("query24") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query25.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query25.groovy
index 30ff2fdd9e3a2f2..ddb841289d2bc78 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query25.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query25.groovy
@@ -28,7 +28,6 @@ suite("query25") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query26.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query26.groovy
index 175449eaabf808d..2319840d268253a 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query26.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query26.groovy
@@ -28,7 +28,6 @@ suite("query26") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query27.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query27.groovy
index e56f8cf0da76d1d..e77293828384b48 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query27.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query27.groovy
@@ -28,7 +28,6 @@ suite("query27") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query28.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query28.groovy
index 09798f18672f807..a5f11f5cdc7c0ff 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query28.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query28.groovy
@@ -28,7 +28,6 @@ suite("query28") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query29.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query29.groovy
index 5af908b5affce15..912a7d42d664268 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query29.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query29.groovy
@@ -28,7 +28,6 @@ suite("query29") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query3.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query3.groovy
index 6fdb1b66e23ffac..0abf3cac4053c5d 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query3.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query3.groovy
@@ -28,7 +28,6 @@ suite("query3") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query30.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query30.groovy
index d3efe50f938964c..39fe1e84466c5bf 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query30.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query30.groovy
@@ -28,7 +28,6 @@ suite("query30") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query31.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query31.groovy
index 3ffeafbdd2d8d74..2d6b992c9e49e14 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query31.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query31.groovy
@@ -28,7 +28,6 @@ suite("query31") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query33.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query33.groovy
index 481706102a76d4a..be0fcd94b354092 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query33.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query33.groovy
@@ -28,7 +28,6 @@ suite("query33") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query34.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query34.groovy
index 9b45eac22662da5..69a42fbd194bbf1 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query34.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query34.groovy
@@ -28,7 +28,6 @@ suite("query34") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query35.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query35.groovy
index b33ddc177f90184..bac67fbfbe42544 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query35.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query35.groovy
@@ -28,7 +28,6 @@ suite("query35") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query36.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query36.groovy
index 57ef14bf8c7f2a2..68dafea1d6074d7 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query36.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query36.groovy
@@ -28,7 +28,6 @@ suite("query36") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query37.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query37.groovy
index 0bbf8043f0f9b25..a4b08d3465a1a67 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query37.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query37.groovy
@@ -28,7 +28,6 @@ suite("query37") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query39.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query39.groovy
index 65f2cd075b8602c..2102336879170cc 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query39.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query39.groovy
@@ -28,7 +28,6 @@ suite("query39") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query4.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query4.groovy
index fa46b393dfafdd3..4f73461b01da0a1 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query4.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query4.groovy
@@ -28,7 +28,6 @@ suite("query4") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query40.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query40.groovy
index 8b6872d5d535dbd..0a1ac5a2ba70526 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query40.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query40.groovy
@@ -28,7 +28,6 @@ suite("query40") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query41.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query41.groovy
index 3da6c31c0c2dbff..29f6098dc7b0afd 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query41.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query41.groovy
@@ -28,7 +28,6 @@ suite("query41") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query42.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query42.groovy
index 4ec2c7009a001a0..5aefcc69b43cace 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query42.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query42.groovy
@@ -28,7 +28,6 @@ suite("query42") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query43.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query43.groovy
index d056a6c83ab1543..b5833db0bb96d36 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query43.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query43.groovy
@@ -28,7 +28,6 @@ suite("query43") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query44.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query44.groovy
index 759568b3d959325..62beee6215e9136 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query44.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query44.groovy
@@ -28,7 +28,6 @@ suite("query44") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query45.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query45.groovy
index db1b0a43a041caf..c5881dcb0495024 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query45.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query45.groovy
@@ -28,7 +28,6 @@ suite("query45") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query46.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query46.groovy
index 9a86138193ef34f..9d0d144e2d33018 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query46.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query46.groovy
@@ -28,7 +28,6 @@ suite("query46") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query47.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query47.groovy
index dd95ab34a4e5071..7709931e387bd9e 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query47.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query47.groovy
@@ -28,7 +28,6 @@ suite("query47") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query48.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query48.groovy
index 8821cdc67bee757..4440407bddacb3f 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query48.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query48.groovy
@@ -28,7 +28,6 @@ suite("query48") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query49.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query49.groovy
index a3564b12470d84a..9ac0dea181db824 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query49.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query49.groovy
@@ -28,7 +28,6 @@ suite("query49") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query5.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query5.groovy
index 9826181eefb89e2..a0edbe4115c2fe3 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query5.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query5.groovy
@@ -28,7 +28,6 @@ suite("query5") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query50.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query50.groovy
index b1334c53128467a..43a930a59cc6609 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query50.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query50.groovy
@@ -28,7 +28,6 @@ suite("query50") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query51.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query51.groovy
index 1994c671f6aea04..073ce567d7ce958 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query51.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query51.groovy
@@ -28,7 +28,6 @@ suite("query51") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query52.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query52.groovy
index 3526d4bfc5bd86b..1d2a64e1a51d88d 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query52.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query52.groovy
@@ -28,7 +28,6 @@ suite("query52") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query53.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query53.groovy
index 086407ef0323cb7..9d4a1d34d1793b8 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query53.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query53.groovy
@@ -28,7 +28,6 @@ suite("query53") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query54.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query54.groovy
index 30ba45278a26157..0c9b1660137f236 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query54.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query54.groovy
@@ -28,7 +28,6 @@ suite("query54") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query55.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query55.groovy
index 74c49a5cb002e65..0d6b359c52cec58 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query55.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query55.groovy
@@ -28,7 +28,6 @@ suite("query55") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query56.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query56.groovy
index 43989b3f4492fcb..aa464c29621dd6b 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query56.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query56.groovy
@@ -28,7 +28,6 @@ suite("query56") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query57.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query57.groovy
index 297666831b38a39..79e7cccd578c41a 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query57.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query57.groovy
@@ -28,7 +28,6 @@ suite("query57") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query58.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query58.groovy
index 11a3950aa4d4440..31b0deeca960543 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query58.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query58.groovy
@@ -28,7 +28,6 @@ suite("query58") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query59.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query59.groovy
index 75318f6ed936a92..850f8ae8e35b977 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query59.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query59.groovy
@@ -28,7 +28,6 @@ suite("query59") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query6.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query6.groovy
index 934f4083ad28f19..05b7af1de05d935 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query6.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query6.groovy
@@ -28,7 +28,6 @@ suite("query6") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query60.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query60.groovy
index 003168fac4c6752..a6817dcf486a78d 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query60.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query60.groovy
@@ -28,7 +28,6 @@ suite("query60") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query61.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query61.groovy
index c5f5aa4e2bfa015..4e571c37a28e9d6 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query61.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query61.groovy
@@ -28,7 +28,6 @@ suite("query61") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query62.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query62.groovy
index 4c628cfa46d98fa..a4f97f5e81eb98b 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query62.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query62.groovy
@@ -28,7 +28,6 @@ suite("query62") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query63.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query63.groovy
index a9e2f6a0c426197..26c08be97939bd6 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query63.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query63.groovy
@@ -28,7 +28,6 @@ suite("query63") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query64.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query64.groovy
index ff1469f001e564b..45bc681e2b33d08 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query64.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query64.groovy
@@ -28,7 +28,6 @@ suite("query64") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query65.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query65.groovy
index f3f6d8288cb9ad7..180471b7864e6fc 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query65.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query65.groovy
@@ -28,7 +28,6 @@ suite("query65") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query66.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query66.groovy
index ecfa4fc6318d497..62ffc10eadf40d7 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query66.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query66.groovy
@@ -28,7 +28,6 @@ suite("query66") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query67.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query67.groovy
index 2ceb61095f7cb44..b39304d2f8a148d 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query67.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query67.groovy
@@ -28,7 +28,6 @@ suite("query67") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query68.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query68.groovy
index ceac65aecb7c9ce..cec85abd8d2b511 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query68.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query68.groovy
@@ -28,7 +28,6 @@ suite("query68") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query69.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query69.groovy
index 831150199ed5b67..e990513c2d2e9bd 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query69.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query69.groovy
@@ -28,7 +28,6 @@ suite("query69") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query7.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query7.groovy
index 51744cf7187bc26..9976350850fa9e3 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query7.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query7.groovy
@@ -28,7 +28,6 @@ suite("query7") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query70.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query70.groovy
index fb66f3af34d84de..80c014e015b5b89 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query70.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query70.groovy
@@ -28,7 +28,6 @@ suite("query70") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query71.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query71.groovy
index 7184928ba72271f..932ffd44a6d363c 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query71.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query71.groovy
@@ -28,7 +28,6 @@ suite("query71") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query72.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query72.groovy
index d2d99b08ba0c3e7..cc8073ac96376e2 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query72.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query72.groovy
@@ -28,7 +28,6 @@ suite("query72") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query73.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query73.groovy
index b93e95ee68bd54b..0669f20301e7ae6 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query73.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query73.groovy
@@ -28,7 +28,6 @@ suite("query73") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query74.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query74.groovy
index 8bff726f931b4d2..34b6082b4d426dd 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query74.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query74.groovy
@@ -28,7 +28,6 @@ suite("query74") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query75.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query75.groovy
index cef2087d1b23ea2..9769556e3e3c373 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query75.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query75.groovy
@@ -28,7 +28,6 @@ suite("query75") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query76.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query76.groovy
index 1edd4f6bc842234..e249b78689c2abc 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query76.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query76.groovy
@@ -28,7 +28,6 @@ suite("query76") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query77.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query77.groovy
index 5256096293f105b..981fc1bd318b96b 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query77.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query77.groovy
@@ -28,7 +28,6 @@ suite("query77") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query78.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query78.groovy
index 474d4b2d70f4c37..074c2441a02885c 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query78.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query78.groovy
@@ -28,7 +28,6 @@ suite("query78") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query79.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query79.groovy
index 7621db3e971f1af..f5153ab7f5f9554 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query79.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query79.groovy
@@ -28,7 +28,6 @@ suite("query79") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query8.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query8.groovy
index c954f1e43a55e7c..98c1c2a9fb52fbd 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query8.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query8.groovy
@@ -28,7 +28,6 @@ suite("query8") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query80.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query80.groovy
index 0b4e93f5e1129ad..b9a5ac4f604c220 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query80.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query80.groovy
@@ -28,7 +28,6 @@ suite("query80") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query81.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query81.groovy
index 05f986b6d962972..6a479e2d1a85e5b 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query81.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query81.groovy
@@ -28,7 +28,6 @@ suite("query81") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query82.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query82.groovy
index e59a0d58c5f5ae3..6eb512ccad004a1 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query82.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query82.groovy
@@ -28,7 +28,6 @@ suite("query82") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query83.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query83.groovy
index b1f0e5dbc791074..f6c516495e2d93b 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query83.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query83.groovy
@@ -28,7 +28,6 @@ suite("query83") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query84.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query84.groovy
index d42d55408a71c53..607fc1809bc0cca 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query84.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query84.groovy
@@ -28,7 +28,6 @@ suite("query84") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query85.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query85.groovy
index a4137feaa50cf34..96d458c33ec41fd 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query85.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query85.groovy
@@ -28,7 +28,6 @@ suite("query85") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query86.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query86.groovy
index ac8b7125ec87250..78291a8b522390a 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query86.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query86.groovy
@@ -28,7 +28,6 @@ suite("query86") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query87.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query87.groovy
index da17575eb9d9111..64e6a8798bf92ff 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query87.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query87.groovy
@@ -28,7 +28,6 @@ suite("query87") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query88.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query88.groovy
index 124ed7d838f016f..ef8ffaaf1a89215 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query88.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query88.groovy
@@ -28,7 +28,6 @@ suite("query88") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query89.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query89.groovy
index 3b6313c1fede1a7..b6f99fb3bbade5d 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query89.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query89.groovy
@@ -28,7 +28,6 @@ suite("query89") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query9.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query9.groovy
index bfba9e3761fc8ca..a516f9e04d22845 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query9.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query9.groovy
@@ -28,7 +28,6 @@ suite("query9") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query90.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query90.groovy
index d03fedd9941a9f9..e3f813df268c27e 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query90.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query90.groovy
@@ -28,7 +28,6 @@ suite("query90") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query91.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query91.groovy
index f3ed0ba0e7955b5..9c73e5cf7d6bfd9 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query91.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query91.groovy
@@ -28,7 +28,6 @@ suite("query91") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query92.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query92.groovy
index 745dcc9021f211d..22442df13460d55 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query92.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query92.groovy
@@ -28,7 +28,6 @@ suite("query92") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query93.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query93.groovy
index c51db4937d00e31..b8a292db37db93d 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query93.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query93.groovy
@@ -28,7 +28,6 @@ suite("query93") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query94.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query94.groovy
index 3d2a7b7c2b31e35..ca522390ad66ca7 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query94.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query94.groovy
@@ -28,7 +28,6 @@ suite("query94") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query95.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query95.groovy
index 4709148cf72e121..cd0c803497463c4 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query95.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query95.groovy
@@ -28,7 +28,6 @@ suite("query95") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query96.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query96.groovy
index 3a9b2d94b52d085..9f662ec70290a73 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query96.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query96.groovy
@@ -28,7 +28,6 @@ suite("query96") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query98.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query98.groovy
index 6f901211477a832..ddfaf00eed1d263 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query98.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query98.groovy
@@ -28,7 +28,6 @@ suite("query98") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query99.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query99.groovy
index 4cd74e913c205fa..7a5b3f7aac0ee6a 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query99.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query99.groovy
@@ -28,7 +28,6 @@ suite("query99") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query1.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query1.groovy
index 6a909215443e869..0ba15a2515bc149 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query1.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query1.groovy
@@ -28,7 +28,6 @@ suite("query1") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query10.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query10.groovy
index 3ed675046f208f9..77bc186bf21cfee 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query10.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query10.groovy
@@ -28,7 +28,6 @@ suite("query10") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query11.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query11.groovy
index f09c688a74921b6..3e36bca6a4ed285 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query11.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query11.groovy
@@ -28,7 +28,6 @@ suite("query11") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query12.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query12.groovy
index e02a36ff855b159..48b964730635383 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query12.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query12.groovy
@@ -28,7 +28,6 @@ suite("query12") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query13.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query13.groovy
index 544b4c5f092d22c..fe0a18676ed7522 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query13.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query13.groovy
@@ -28,7 +28,6 @@ suite("query13") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query14.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query14.groovy
index abbedc633124bb8..d7fce388f00a0ee 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query14.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query14.groovy
@@ -28,7 +28,6 @@ suite("query14") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query15.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query15.groovy
index 32d67dfeb6f7b5e..8b540d67092b54f 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query15.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query15.groovy
@@ -28,7 +28,6 @@ suite("query15") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query16.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query16.groovy
index c4619a55e6b8de5..4a2805bf5d0c34d 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query16.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query16.groovy
@@ -28,7 +28,6 @@ suite("query16") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query17.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query17.groovy
index 777ed07eaf070fa..251425bc807c556 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query17.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query17.groovy
@@ -28,7 +28,6 @@ suite("query17") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query18.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query18.groovy
index 6358b59c05fb218..d1dfee49fbc05da 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query18.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query18.groovy
@@ -28,7 +28,6 @@ suite("query18") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query19.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query19.groovy
index b9cb617dfbf1708..721e48843058b67 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query19.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query19.groovy
@@ -28,7 +28,6 @@ suite("query19") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query2.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query2.groovy
index 4fe3ae8c7156bfd..2c96a1c32158801 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query2.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query2.groovy
@@ -28,7 +28,6 @@ suite("query2") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query20.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query20.groovy
index 167c468e5a5079a..455cb2ffa05385b 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query20.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query20.groovy
@@ -28,7 +28,6 @@ suite("query20") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query21.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query21.groovy
index d278855fc74d178..156a17fe85b6ec1 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query21.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query21.groovy
@@ -29,7 +29,6 @@ suite("query21") {
     sql 'SET enable_fold_constant_by_be = false' //plan shape will be different
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query22.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query22.groovy
index be67f4f810c6a32..f631cf27eb850e0 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query22.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query22.groovy
@@ -28,7 +28,6 @@ suite("query22") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query24.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query24.groovy
index 0904f936cf4dc79..193bfc369422bc8 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query24.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query24.groovy
@@ -28,7 +28,6 @@ suite("query24") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query25.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query25.groovy
index a0b3efed5f8df4a..7e760bd031866cd 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query25.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query25.groovy
@@ -28,7 +28,6 @@ suite("query25") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query26.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query26.groovy
index 2b5b77fbcc32bec..81725ec5a3e1523 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query26.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query26.groovy
@@ -28,7 +28,6 @@ suite("query26") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query27.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query27.groovy
index e17d19979afc938..b5e19f300c4c432 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query27.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query27.groovy
@@ -28,7 +28,6 @@ suite("query27") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query28.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query28.groovy
index 370386cc47991e8..5bc6151a2da57b5 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query28.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query28.groovy
@@ -28,7 +28,6 @@ suite("query28") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query29.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query29.groovy
index 44425d7038272cd..ca459befde8333a 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query29.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query29.groovy
@@ -28,7 +28,6 @@ suite("query29") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query3.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query3.groovy
index 8f56818cc2af56d..380ad2b851ff4d6 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query3.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query3.groovy
@@ -28,7 +28,6 @@ suite("query3") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query30.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query30.groovy
index 3c803662b5b3bd4..8896f5f6376caa3 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query30.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query30.groovy
@@ -28,7 +28,6 @@ suite("query30") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query31.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query31.groovy
index a1b8a9277d7329d..0abc28d0096f975 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query31.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query31.groovy
@@ -28,7 +28,6 @@ suite("query31") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query33.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query33.groovy
index 8531b0fcf8d7d4f..4e939a1f625df2a 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query33.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query33.groovy
@@ -28,7 +28,6 @@ suite("query33") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query34.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query34.groovy
index baf93402158ee73..9da8d0a2407257d 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query34.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query34.groovy
@@ -28,7 +28,6 @@ suite("query34") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query35.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query35.groovy
index 9d8ccc9e339a3df..9dd1c8fec1262f2 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query35.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query35.groovy
@@ -28,7 +28,6 @@ suite("query35") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query36.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query36.groovy
index d3396d1982f4a6d..92d56da68a4b143 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query36.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query36.groovy
@@ -28,7 +28,6 @@ suite("query36") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query37.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query37.groovy
index 45fe042f50455ab..46732fd29e94d15 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query37.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query37.groovy
@@ -28,7 +28,6 @@ suite("query37") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query39.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query39.groovy
index 9a75f8af4ff5348..89b17019a4e94a7 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query39.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query39.groovy
@@ -28,7 +28,6 @@ suite("query39") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query4.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query4.groovy
index d181c572cb4c471..f58d723809ef82e 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query4.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query4.groovy
@@ -28,7 +28,6 @@ suite("query4") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query40.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query40.groovy
index 163acc8ef69f582..ad9e2edb82a07c0 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query40.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query40.groovy
@@ -28,7 +28,6 @@ suite("query40") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query41.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query41.groovy
index f44834cbc7043da..8a84c0ee387bad1 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query41.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query41.groovy
@@ -28,7 +28,6 @@ suite("query41") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query42.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query42.groovy
index bba3bdf9a8eae57..a653f8d4df51454 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query42.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query42.groovy
@@ -28,7 +28,6 @@ suite("query42") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query43.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query43.groovy
index b4a846e09120601..47f09999b526ead 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query43.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query43.groovy
@@ -28,7 +28,6 @@ suite("query43") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query44.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query44.groovy
index 3ae04b32990ddc1..710c3a958039288 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query44.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query44.groovy
@@ -28,7 +28,6 @@ suite("query44") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query45.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query45.groovy
index 4c412d942fdad03..64d23d1b10c49ca 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query45.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query45.groovy
@@ -28,7 +28,6 @@ suite("query45") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query46.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query46.groovy
index f0f56e556ef41e3..af5d48446d60eb5 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query46.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query46.groovy
@@ -28,7 +28,6 @@ suite("query46") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query47.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query47.groovy
index 0ed22d0d4b3bed1..a3dab5e6a9df425 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query47.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query47.groovy
@@ -28,7 +28,6 @@ suite("query47") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query48.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query48.groovy
index b18cd2d23b3daa2..c0ea96fcbcb1af2 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query48.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query48.groovy
@@ -28,7 +28,6 @@ suite("query48") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query49.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query49.groovy
index c8a166e98b9f82d..421aee236b36088 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query49.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query49.groovy
@@ -28,7 +28,6 @@ suite("query49") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query5.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query5.groovy
index 09d9b7d0d5904bc..c8878f58eede172 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query5.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query5.groovy
@@ -28,7 +28,6 @@ suite("query5") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query50.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query50.groovy
index a7b1d2e9160b82e..ee1e6a242f1d246 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query50.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query50.groovy
@@ -28,7 +28,6 @@ suite("query50") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query51.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query51.groovy
index bbc9b541c6d1b5b..6ba16b7df8cb158 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query51.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query51.groovy
@@ -28,7 +28,6 @@ suite("query51") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query52.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query52.groovy
index 9fce8bac2b31915..39921cafa767346 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query52.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query52.groovy
@@ -28,7 +28,6 @@ suite("query52") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query53.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query53.groovy
index a88478a207f1a59..83d37f6eba9e38f 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query53.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query53.groovy
@@ -28,7 +28,6 @@ suite("query53") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query54.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query54.groovy
index 44e504f42945cb0..74d91188a4d0b93 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query54.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query54.groovy
@@ -28,7 +28,6 @@ suite("query54") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query55.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query55.groovy
index cfe0df5604d8023..0b55c0e44292dd4 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query55.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query55.groovy
@@ -28,7 +28,6 @@ suite("query55") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query56.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query56.groovy
index 74ef30cb14958a7..cc20b3c582ec2ab 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query56.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query56.groovy
@@ -28,7 +28,6 @@ suite("query56") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query57.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query57.groovy
index e4a16668c66dedc..bbf0d3672bb1c42 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query57.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query57.groovy
@@ -28,7 +28,6 @@ suite("query57") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query58.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query58.groovy
index 6fd5fcdbbab86ab..5bcd99e0158927f 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query58.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query58.groovy
@@ -28,7 +28,6 @@ suite("query58") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query59.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query59.groovy
index 86f1f10abee42e2..436cbb0cb7daca2 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query59.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query59.groovy
@@ -28,7 +28,6 @@ suite("query59") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query6.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query6.groovy
index d76f10a6ac1229f..c8ea2880893f147 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query6.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query6.groovy
@@ -28,7 +28,6 @@ suite("query6") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query60.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query60.groovy
index f64dc4757cec7de..51ce493d8b2d3ce 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query60.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query60.groovy
@@ -28,7 +28,6 @@ suite("query60") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query61.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query61.groovy
index ed76eae2f7da0c1..80086eeb91b312a 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query61.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query61.groovy
@@ -28,7 +28,6 @@ suite("query61") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query62.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query62.groovy
index 58410fc549b6400..549493651ba701d 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query62.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query62.groovy
@@ -28,7 +28,6 @@ suite("query62") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query63.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query63.groovy
index 8397c58e83d9e69..3cb7a14846a3fe1 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query63.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query63.groovy
@@ -28,7 +28,6 @@ suite("query63") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query64.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query64.groovy
index 69418de37f1c4a7..982f1d486f70e20 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query64.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query64.groovy
@@ -28,7 +28,6 @@ suite("query64") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query65.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query65.groovy
index 101788876ad5fe6..7ba24fea06efa3c 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query65.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query65.groovy
@@ -28,7 +28,6 @@ suite("query65") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query66.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query66.groovy
index c625dad30854def..9ac4430fe31fe1a 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query66.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query66.groovy
@@ -28,7 +28,6 @@ suite("query66") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query67.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query67.groovy
index 12647173321cf00..04549eafc91fd39 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query67.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query67.groovy
@@ -28,7 +28,6 @@ suite("query67") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query68.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query68.groovy
index 8aabd2d324b15e3..1ee08e693473c4f 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query68.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query68.groovy
@@ -28,7 +28,6 @@ suite("query68") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query69.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query69.groovy
index 2814cefea885b22..505233e8a6a32d4 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query69.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query69.groovy
@@ -28,7 +28,6 @@ suite("query69") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query7.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query7.groovy
index 782cd71565c4562..5f9b9fb56e40a49 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query7.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query7.groovy
@@ -28,7 +28,6 @@ suite("query7") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query70.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query70.groovy
index d9d6538303427c9..95c6e377302c9a9 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query70.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query70.groovy
@@ -28,7 +28,6 @@ suite("query70") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query71.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query71.groovy
index e33b27c8bc15877..4923dad4f29b095 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query71.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query71.groovy
@@ -28,7 +28,6 @@ suite("query71") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query72.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query72.groovy
index e73692784f3dc0c..b6ba5e333302863 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query72.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query72.groovy
@@ -28,7 +28,6 @@ suite("query72") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query73.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query73.groovy
index b5059efac308426..948371a5ffcc395 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query73.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query73.groovy
@@ -28,7 +28,6 @@ suite("query73") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query74.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query74.groovy
index e9c0b18b6f0a3ba..ef2c9c8f8d3d7cf 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query74.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query74.groovy
@@ -28,7 +28,6 @@ suite("query74") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query75.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query75.groovy
index 2aa5abf89a65b03..0fecb5d5b1d9c4f 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query75.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query75.groovy
@@ -28,7 +28,6 @@ suite("query75") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query76.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query76.groovy
index 4e357fa7f77ff29..3f0a2b9677f3330 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query76.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query76.groovy
@@ -28,7 +28,6 @@ suite("query76") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query77.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query77.groovy
index 29ff511811cfde7..062fee510b5a8a2 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query77.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query77.groovy
@@ -28,7 +28,6 @@ suite("query77") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query78.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query78.groovy
index 52dea88de569364..d7d257876e1636b 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query78.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query78.groovy
@@ -28,7 +28,6 @@ suite("query78") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query79.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query79.groovy
index 8d2bb23d2ba5976..0bdf6c59a887984 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query79.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query79.groovy
@@ -28,7 +28,6 @@ suite("query79") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query8.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query8.groovy
index 2c2d83ebc23e8fe..aaf086736c541cb 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query8.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query8.groovy
@@ -28,7 +28,6 @@ suite("query8") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query80.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query80.groovy
index e43ace5ebab5924..154d2dcea1877bf 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query80.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query80.groovy
@@ -28,7 +28,6 @@ suite("query80") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query81.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query81.groovy
index 2967c311dd7759a..2ffa0c635ae232f 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query81.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query81.groovy
@@ -28,7 +28,6 @@ suite("query81") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query82.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query82.groovy
index afe984f39e0171b..61653a4628594c9 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query82.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query82.groovy
@@ -28,7 +28,6 @@ suite("query82") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query83.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query83.groovy
index 450db33a73dd005..9716bb3a7fa5f42 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query83.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query83.groovy
@@ -28,7 +28,6 @@ suite("query83") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query84.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query84.groovy
index bb01a254dc05d2f..ed62df9d52a3eb0 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query84.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query84.groovy
@@ -28,7 +28,6 @@ suite("query84") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query85.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query85.groovy
index 6458f47d0d37aec..52ce4c171fff225 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query85.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query85.groovy
@@ -28,7 +28,6 @@ suite("query85") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query86.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query86.groovy
index e607fbbfefc1d77..25116794a42d9b0 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query86.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query86.groovy
@@ -28,7 +28,6 @@ suite("query86") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query87.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query87.groovy
index a5a4826500afdfd..20db01a89674920 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query87.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query87.groovy
@@ -28,7 +28,6 @@ suite("query87") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query88.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query88.groovy
index d81dd616d59769f..1b3e50cabab387e 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query88.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query88.groovy
@@ -28,7 +28,6 @@ suite("query88") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query89.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query89.groovy
index a0c8fa972a57266..a7e34fc464a1f0d 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query89.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query89.groovy
@@ -28,7 +28,6 @@ suite("query89") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query9.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query9.groovy
index dd71506e1bd90c3..78ab98b2dd541c1 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query9.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query9.groovy
@@ -28,7 +28,6 @@ suite("query9") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query90.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query90.groovy
index 99dbb7f75ef3c49..50a849e8a1ecffc 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query90.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query90.groovy
@@ -28,7 +28,6 @@ suite("query90") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query91.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query91.groovy
index 800d1a12d1a5873..b9ba59d7a71bc0e 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query91.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query91.groovy
@@ -28,7 +28,6 @@ suite("query91") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query92.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query92.groovy
index 70b74051f039c62..2f9f54476f91a97 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query92.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query92.groovy
@@ -28,7 +28,6 @@ suite("query92") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query93.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query93.groovy
index e88a06cfb3a3870..9e12335699b170b 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query93.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query93.groovy
@@ -28,7 +28,6 @@ suite("query93") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query94.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query94.groovy
index 29307f0f46681ee..1e1b677b20adca0 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query94.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query94.groovy
@@ -28,7 +28,6 @@ suite("query94") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query95.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query95.groovy
index 6f126d5ffd7a7ea..8e17125bdc244d8 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query95.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query95.groovy
@@ -28,7 +28,6 @@ suite("query95") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query96.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query96.groovy
index c21be5c6856a8a3..1c62dc6b986cfd3 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query96.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query96.groovy
@@ -28,7 +28,6 @@ suite("query96") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query98.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query98.groovy
index 409bd1f1e9271c9..3c7a9ba360f8b82 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query98.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query98.groovy
@@ -28,7 +28,6 @@ suite("query98") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query99.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query99.groovy
index 599582d2f98ae40..dcc24829c8d0079 100644
--- a/regression-test/suites/shape_check/tpcds_sf100/shape/query99.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf100/shape/query99.groovy
@@ -28,7 +28,6 @@ suite("query99") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query13.groovy b/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query13.groovy
index d8c82713b6f24a3..441ffe2d3027cef 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query13.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query13.groovy
@@ -28,7 +28,6 @@ suite("query13") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query19.groovy b/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query19.groovy
index 71d224ca8bb14c6..ace096e14ea18d7 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query19.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query19.groovy
@@ -28,7 +28,6 @@ suite("query19") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query44.groovy b/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query44.groovy
index cbfccc442bbd4b6..29f73caa2b6c4de 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query44.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query44.groovy
@@ -28,7 +28,6 @@ suite("query44") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query45.groovy b/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query45.groovy
index 0f34ea3603e4e7c..667b39b6dc97f30 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query45.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query45.groovy
@@ -28,7 +28,6 @@ suite("query45") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query54.groovy b/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query54.groovy
index 6aa1af6448a432e..fc38b62cae73872 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query54.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query54.groovy
@@ -28,7 +28,6 @@ suite("query54") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query56.groovy b/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query56.groovy
index 6a4d8acb78706cf..c1bf19a49079ddb 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query56.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query56.groovy
@@ -28,7 +28,6 @@ suite("query56") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query6.groovy b/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query6.groovy
index d4c586c151a897e..8d36c9a6e65cf15 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query6.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query6.groovy
@@ -28,7 +28,6 @@ suite("query6") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query61.groovy b/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query61.groovy
index 685edbb0c9867dd..c2adc6f24bbbfac 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query61.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query61.groovy
@@ -28,7 +28,6 @@ suite("query61") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query68.groovy b/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query68.groovy
index cfce17ece894a5b..5de25c2a689ea9d 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query68.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query68.groovy
@@ -28,7 +28,6 @@ suite("query68") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query8.groovy b/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query8.groovy
index 5a138f424fb2ff4..4cbbc63ca53b0b2 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query8.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query8.groovy
@@ -28,7 +28,6 @@ suite("query8") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query91.groovy b/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query91.groovy
index fc527d703687bf7..e5b9a4177da98ed 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query91.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query91.groovy
@@ -28,7 +28,6 @@ suite("query91") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query95.groovy b/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query95.groovy
index 5afab1b1c05e4db..b3b2dec53beeb84 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query95.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/bs_downgrade_shape/query95.groovy
@@ -28,7 +28,6 @@ suite("query95") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/ddl/shape.tmpl b/regression-test/suites/shape_check/tpcds_sf1000/ddl/shape.tmpl
index d68083c4ff1997d..0e849d3a25f2fee 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/ddl/shape.tmpl
+++ b/regression-test/suites/shape_check/tpcds_sf1000/ddl/shape.tmpl
@@ -28,7 +28,6 @@ suite("query{--}") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/eliminate_empty/query10_empty.groovy b/regression-test/suites/shape_check/tpcds_sf1000/eliminate_empty/query10_empty.groovy
index 5e53ca3d697cfd8..daec20579613cae 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/eliminate_empty/query10_empty.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/eliminate_empty/query10_empty.groovy
@@ -29,7 +29,6 @@ suite("query10") {
       set enable_fallback_to_original_planner=false;
       set exec_mem_limit=21G;
       set be_number_for_test=3;
-      set parallel_fragment_exec_instance_num=8;
       set parallel_pipeline_task_num=8;
       set forbid_unknown_col_stats=true;
       set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query1.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query1.groovy
index 92cac960df695eb..5a4eff5b7fc4bb6 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query1.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query1.groovy
@@ -28,7 +28,6 @@ suite("query1") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query10.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query10.groovy
index f9c6d80b84f704a..51b9b8e9065ed75 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query10.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query10.groovy
@@ -28,7 +28,6 @@ suite("query10") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query11.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query11.groovy
index a99c0fb2844909a..d24783216dd7d8d 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query11.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query11.groovy
@@ -28,7 +28,6 @@ suite("query11") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query12.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query12.groovy
index db173ea3d81a451..bbe9894bd5b3104 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query12.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query12.groovy
@@ -28,7 +28,6 @@ suite("query12") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query13.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query13.groovy
index e6850d187a5c6db..4574d0e1028d4d7 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query13.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query13.groovy
@@ -28,7 +28,6 @@ suite("query13") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query14.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query14.groovy
index d508d7ba1a586cf..519ca8dccf91df0 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query14.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query14.groovy
@@ -28,7 +28,6 @@ suite("query14") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query15.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query15.groovy
index e244c34686be9ec..59b8a7e8aa2fca7 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query15.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query15.groovy
@@ -28,7 +28,6 @@ suite("query15") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query16.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query16.groovy
index 90e5de42bdb5aac..60575174c1dce59 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query16.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query16.groovy
@@ -28,7 +28,6 @@ suite("query16") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query17.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query17.groovy
index 779f982558c90f8..91caf8193b48877 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query17.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query17.groovy
@@ -28,7 +28,6 @@ suite("query17") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query18.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query18.groovy
index ae0ea359c1d09c4..eb69c41143b7f5f 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query18.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query18.groovy
@@ -28,7 +28,6 @@ suite("query18") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query19.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query19.groovy
index 70aa3e4f4131da9..4a6461973b5fd82 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query19.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query19.groovy
@@ -28,7 +28,6 @@ suite("query19") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query2.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query2.groovy
index 50ffb28aeb41b76..7b223b0e613fd6e 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query2.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query2.groovy
@@ -28,7 +28,6 @@ suite("query2") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query20.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query20.groovy
index 419e9fc593ada43..71cd3f870094181 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query20.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query20.groovy
@@ -28,7 +28,6 @@ suite("query20") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query21.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query21.groovy
index 9c50c722a73ac38..e9bd5fa5a1a5a33 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query21.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query21.groovy
@@ -29,7 +29,6 @@ suite("query21") {
     sql 'SET enable_fold_constant_by_be = false' //plan shape will be different
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query22.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query22.groovy
index 5ef9a0bbbbcce22..4c26e8ae7b1664c 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query22.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query22.groovy
@@ -28,7 +28,6 @@ suite("query22") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query24.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query24.groovy
index 2f3d5b950e9cc35..79e78efd914062f 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query24.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query24.groovy
@@ -28,7 +28,6 @@ suite("query24") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query25.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query25.groovy
index 19d4ef8443f9529..43ccfb1b943b86d 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query25.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query25.groovy
@@ -28,7 +28,6 @@ suite("query25") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query26.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query26.groovy
index c71bb92aac72ebe..d123007a8a8c9ae 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query26.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query26.groovy
@@ -28,7 +28,6 @@ suite("query26") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query27.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query27.groovy
index ec5f4ac2f1ad93e..bbad86218fb5ecb 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query27.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query27.groovy
@@ -28,7 +28,6 @@ suite("query27") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query28.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query28.groovy
index 8c43bbaf31144aa..a8151135102b1c7 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query28.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query28.groovy
@@ -28,7 +28,6 @@ suite("query28") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query29.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query29.groovy
index f8c7f94cf071708..737449cb9ad7b83 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query29.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query29.groovy
@@ -28,7 +28,6 @@ suite("query29") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query3.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query3.groovy
index db9d172dfb4cfa7..5642f8b8193d0fb 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query3.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query3.groovy
@@ -28,7 +28,6 @@ suite("query3") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query30.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query30.groovy
index 0d89beaab9f0c2e..6a208af9815b410 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query30.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query30.groovy
@@ -28,7 +28,6 @@ suite("query30") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query31.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query31.groovy
index 06d3717ca0600f2..1c5439b2f483ee9 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query31.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query31.groovy
@@ -28,7 +28,6 @@ suite("query31") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query34.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query34.groovy
index 79972f33bb7ee01..aa07b41f22ccb65 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query34.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query34.groovy
@@ -28,7 +28,6 @@ suite("query34") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query36.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query36.groovy
index 580f025d597eeec..64067ab0fde0b83 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query36.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query36.groovy
@@ -28,7 +28,6 @@ suite("query36") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query37.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query37.groovy
index c28de3ae6e6a136..3d46937b21b5734 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query37.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query37.groovy
@@ -28,7 +28,6 @@ suite("query37") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query39.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query39.groovy
index 1a6d2fa5e2ee4a3..0363a8349ca9445 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query39.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query39.groovy
@@ -28,7 +28,6 @@ suite("query39") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query4.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query4.groovy
index 2ed84768d45544c..50e1ed4ac51255d 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query4.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query4.groovy
@@ -28,7 +28,6 @@ suite("query4") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query40.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query40.groovy
index 9daa738d8bf42b6..9eb864f7c4ce97b 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query40.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query40.groovy
@@ -28,7 +28,6 @@ suite("query40") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query41.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query41.groovy
index 8cd49d695fc938a..b9c4a9231e533c2 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query41.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query41.groovy
@@ -28,7 +28,6 @@ suite("query41") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query42.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query42.groovy
index 9a368e763ecc1b5..53a915b0256aaee 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query42.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query42.groovy
@@ -28,7 +28,6 @@ suite("query42") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query43.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query43.groovy
index ec20d53433519e1..6316c19aa0a5618 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query43.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query43.groovy
@@ -28,7 +28,6 @@ suite("query43") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query44.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query44.groovy
index a6804718503b464..fbe34de0da86707 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query44.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query44.groovy
@@ -28,7 +28,6 @@ suite("query44") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query45.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query45.groovy
index 3a22a33b5953092..ee73c4d7287a413 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query45.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query45.groovy
@@ -28,7 +28,6 @@ suite("query45") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query46.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query46.groovy
index 3f255f8c6be36bf..45e4027e8250cee 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query46.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query46.groovy
@@ -28,7 +28,6 @@ suite("query46") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query47.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query47.groovy
index 200125adf9e845f..c0f577a1bcb7553 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query47.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query47.groovy
@@ -28,7 +28,6 @@ suite("query47") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query48.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query48.groovy
index f44e3f3ac46e8fa..31fb1a34d534b95 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query48.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query48.groovy
@@ -28,7 +28,6 @@ suite("query48") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query49.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query49.groovy
index 148c9f6fbfa7a2c..a250ee0abc49cc8 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query49.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query49.groovy
@@ -28,7 +28,6 @@ suite("query49") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query5.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query5.groovy
index 3232b380eade74f..e58e16255cb34be 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query5.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query5.groovy
@@ -28,7 +28,6 @@ suite("query5") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query50.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query50.groovy
index 77771941874530e..f9ac5c26d64a064 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query50.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query50.groovy
@@ -28,7 +28,6 @@ suite("query50") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query51.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query51.groovy
index a8e9d1b590d3cda..9c86cc5447f3770 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query51.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query51.groovy
@@ -28,7 +28,6 @@ suite("query51") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query52.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query52.groovy
index 4735f2cc493044c..9d54f13f391b167 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query52.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query52.groovy
@@ -28,7 +28,6 @@ suite("query52") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query53.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query53.groovy
index e44ccfed048967e..663e12cebf211d6 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query53.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query53.groovy
@@ -28,7 +28,6 @@ suite("query53") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query54.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query54.groovy
index 8918b599a3d4ca6..9c614cfb7680f33 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query54.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query54.groovy
@@ -28,7 +28,6 @@ suite("query54") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query55.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query55.groovy
index 976dfaa71e1380e..cc7e02f4a82f58c 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query55.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query55.groovy
@@ -28,7 +28,6 @@ suite("query55") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query56.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query56.groovy
index a311cf8771405b4..2aaceceedb6e45b 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query56.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query56.groovy
@@ -28,7 +28,6 @@ suite("query56") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query57.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query57.groovy
index d344c350cc55934..4989bdbd0a8ae9e 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query57.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query57.groovy
@@ -28,7 +28,6 @@ suite("query57") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query58.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query58.groovy
index 27238e51637d2ef..4424f4ca810cac8 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query58.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query58.groovy
@@ -28,7 +28,6 @@ suite("query58") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query59.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query59.groovy
index c7df47907a3eb72..b087b8f72814f06 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query59.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query59.groovy
@@ -28,7 +28,6 @@ suite("query59") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query6.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query6.groovy
index ff89e5b12b621f7..b1a51aa274d4f52 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query6.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query6.groovy
@@ -28,7 +28,6 @@ suite("query6") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query60.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query60.groovy
index ddcd041e001be09..f323b96ed5683ae 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query60.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query60.groovy
@@ -28,7 +28,6 @@ suite("query60") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query61.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query61.groovy
index 916b9b686c82d29..784bad5649e4861 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query61.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query61.groovy
@@ -28,7 +28,6 @@ suite("query61") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query62.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query62.groovy
index 6d143dabec5686e..62dae049a193f0a 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query62.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query62.groovy
@@ -28,7 +28,6 @@ suite("query62") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query63.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query63.groovy
index d056b177e2bb9be..a6d5a9ab15781ff 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query63.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query63.groovy
@@ -28,7 +28,6 @@ suite("query63") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query64.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query64.groovy
index 027fd1b9960b5e6..1bdd874bb601ccf 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query64.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query64.groovy
@@ -28,7 +28,6 @@ suite("query64") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query65.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query65.groovy
index a174c367df29936..a6a557012acd66e 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query65.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query65.groovy
@@ -28,7 +28,6 @@ suite("query65") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query66.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query66.groovy
index c55b1e7298d8440..3ed91231c6c2a51 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query66.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query66.groovy
@@ -28,7 +28,6 @@ suite("query66") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query67.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query67.groovy
index 4819075c907c4d5..262746bc1204cf4 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query67.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query67.groovy
@@ -28,7 +28,6 @@ suite("query67") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query68.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query68.groovy
index fc2debddbf87b62..f7dcd02d6d18240 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query68.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query68.groovy
@@ -28,7 +28,6 @@ suite("query68") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query69.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query69.groovy
index 8037d49d7579a61..dafcb98e9265aa7 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query69.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query69.groovy
@@ -28,7 +28,6 @@ suite("query69") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query7.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query7.groovy
index 205a9f191250dff..f65f411d0037a93 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query7.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query7.groovy
@@ -28,7 +28,6 @@ suite("query7") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query70.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query70.groovy
index d3751ca94af8cb7..b0b3583e1cd58d6 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query70.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query70.groovy
@@ -28,7 +28,6 @@ suite("query70") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query71.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query71.groovy
index 2e5e1ccb85b6ee4..21d2167ca139942 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query71.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query71.groovy
@@ -28,7 +28,6 @@ suite("query71") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query72.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query72.groovy
index ba38ec630f24e9b..79a20132cf251f0 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query72.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query72.groovy
@@ -28,7 +28,6 @@ suite("query72") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query73.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query73.groovy
index c48eb2e748875e0..9f24ac2476185b5 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query73.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query73.groovy
@@ -28,7 +28,6 @@ suite("query73") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query74.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query74.groovy
index 944e97c1ef1f577..a0ef28422cd655e 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query74.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query74.groovy
@@ -28,7 +28,6 @@ suite("query74") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query75.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query75.groovy
index b91fbb54fa8194a..5109f9a2acbbe30 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query75.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query75.groovy
@@ -28,7 +28,6 @@ suite("query75") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query76.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query76.groovy
index c83d452b0545e65..e4ab83d22d61efc 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query76.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query76.groovy
@@ -28,7 +28,6 @@ suite("query76") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query77.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query77.groovy
index 6100f7d42a8a85a..3e5fada6bb4aefb 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query77.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query77.groovy
@@ -28,7 +28,6 @@ suite("query77") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query78.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query78.groovy
index 3d8fa8541413361..c862dba5aee05a3 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query78.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query78.groovy
@@ -28,7 +28,6 @@ suite("query78") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query79.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query79.groovy
index d4a0140eb6f866d..200d398fc4123b4 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query79.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query79.groovy
@@ -28,7 +28,6 @@ suite("query79") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query8.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query8.groovy
index ae733a9d404418a..df46b667dea8910 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query8.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query8.groovy
@@ -28,7 +28,6 @@ suite("query8") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query80.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query80.groovy
index 57fd016217d51b6..84cd9a1c1321962 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query80.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query80.groovy
@@ -28,7 +28,6 @@ suite("query80") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query81.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query81.groovy
index 8fffbf8cb13a592..8975f32610d77a2 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query81.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query81.groovy
@@ -28,7 +28,6 @@ suite("query81") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query82.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query82.groovy
index 89bbce3fcaf47fe..b71a3da65ea507c 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query82.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query82.groovy
@@ -28,7 +28,6 @@ suite("query82") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query84.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query84.groovy
index 9e08f9874de0124..e56477699ab1789 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query84.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query84.groovy
@@ -28,7 +28,6 @@ suite("query84") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query85.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query85.groovy
index ae21cca9e688c79..18089c065ecd38b 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query85.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query85.groovy
@@ -28,7 +28,6 @@ suite("query85") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query86.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query86.groovy
index 1756297cd401e41..9143f4f0a9d807d 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query86.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query86.groovy
@@ -28,7 +28,6 @@ suite("query86") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query87.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query87.groovy
index 664c930f03f32b5..f7a38a963c5c75d 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query87.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query87.groovy
@@ -28,7 +28,6 @@ suite("query87") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query88.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query88.groovy
index 5517225e390144e..ab6dfae3788b2aa 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query88.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query88.groovy
@@ -28,7 +28,6 @@ suite("query88") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query89.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query89.groovy
index 50a936d91ec2cfa..cda0b47ee2a810c 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query89.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query89.groovy
@@ -28,7 +28,6 @@ suite("query89") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query9.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query9.groovy
index 63359a154eda53b..fbd7549e41034c9 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query9.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query9.groovy
@@ -28,7 +28,6 @@ suite("query9") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query90.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query90.groovy
index 3e8dd027c8a0fd4..f3ea9018864c104 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query90.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query90.groovy
@@ -28,7 +28,6 @@ suite("query90") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query91.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query91.groovy
index cf8c30e1f874666..e74b45b19e6b618 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query91.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query91.groovy
@@ -28,7 +28,6 @@ suite("query91") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query92.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query92.groovy
index 211dda8ed160df0..bbb560e3393298b 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query92.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query92.groovy
@@ -28,7 +28,6 @@ suite("query92") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query93.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query93.groovy
index 513447b1bc0f5df..53d51c84cae9c80 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query93.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query93.groovy
@@ -28,7 +28,6 @@ suite("query93") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query94.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query94.groovy
index a0ca4799a1520b6..a3bfc5466a61c44 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query94.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query94.groovy
@@ -28,7 +28,6 @@ suite("query94") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query95.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query95.groovy
index 3e32c96597d5dc3..dbb8d1424f52a72 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query95.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query95.groovy
@@ -28,7 +28,6 @@ suite("query95") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query96.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query96.groovy
index df1fe47d7689051..d06dc55b12f467d 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query96.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query96.groovy
@@ -28,7 +28,6 @@ suite("query96") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query98.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query98.groovy
index 348379bef2af16d..4c4d59aac0126c1 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query98.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query98.groovy
@@ -28,7 +28,6 @@ suite("query98") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query99.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query99.groovy
index b1f2ac7ab311f9b..5d651d6d5b10474 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query99.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/hint/query99.groovy
@@ -28,7 +28,6 @@ suite("query99") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query1.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query1.groovy
index 4b7bb59435996e5..914a0af2189b842 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query1.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query1.groovy
@@ -28,7 +28,6 @@ suite("query1") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query10.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query10.groovy
index d075fb0e78b16f8..ea2227510071401 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query10.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query10.groovy
@@ -28,7 +28,6 @@ suite("query10") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query11.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query11.groovy
index 8f817e45a8ed5a0..09086cf80e9f82f 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query11.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query11.groovy
@@ -28,7 +28,6 @@ suite("query11") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query12.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query12.groovy
index d849378f0ebfe76..ae0b8e7452158e8 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query12.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query12.groovy
@@ -28,7 +28,6 @@ suite("query12") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query13.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query13.groovy
index dd75ba6c448a9bb..55883a36ae7e95d 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query13.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query13.groovy
@@ -28,7 +28,6 @@ suite("query13") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query14.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query14.groovy
index 663bb4c1576cadc..753fec2e26658f8 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query14.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query14.groovy
@@ -28,7 +28,6 @@ suite("query14") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query15.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query15.groovy
index e24f29a8d07555b..5ae2eb30d1ee1cc 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query15.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query15.groovy
@@ -28,7 +28,6 @@ suite("query15") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query16.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query16.groovy
index 4b0ba017f9fa089..2e8c4098fa2e85e 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query16.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query16.groovy
@@ -28,7 +28,6 @@ suite("query16") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query17.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query17.groovy
index 3d729082ca042ab..1c973e7d09aadc4 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query17.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query17.groovy
@@ -28,7 +28,6 @@ suite("query17") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query18.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query18.groovy
index d42f277701653a8..8592f99318d1953 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query18.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query18.groovy
@@ -28,7 +28,6 @@ suite("query18") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query19.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query19.groovy
index 8c901bce05e8550..bb1dca6574e40da 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query19.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query19.groovy
@@ -28,7 +28,6 @@ suite("query19") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query2.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query2.groovy
index abfdd4fb44a21ca..7ae18743e607742 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query2.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query2.groovy
@@ -28,7 +28,6 @@ suite("query2") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query20.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query20.groovy
index 49917d5fc11ea12..914be027b5b4d89 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query20.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query20.groovy
@@ -28,7 +28,6 @@ suite("query20") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query21.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query21.groovy
index c421e7d61b4bf01..b8e7943e7fd265d 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query21.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query21.groovy
@@ -29,7 +29,6 @@ suite("query21") {
     sql 'SET enable_fold_constant_by_be = false' //plan shape will be different
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query22.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query22.groovy
index 8485655c939a45b..28907748d9b2ba1 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query22.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query22.groovy
@@ -28,7 +28,6 @@ suite("query22") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query24.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query24.groovy
index 1c9ca97e38caf3a..07aab6781c5a5c5 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query24.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query24.groovy
@@ -28,7 +28,6 @@ suite("query24") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query25.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query25.groovy
index 67cc8f492a07e2f..f9b37733278bc4f 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query25.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query25.groovy
@@ -28,7 +28,6 @@ suite("query25") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query26.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query26.groovy
index 25d6adc6b4e809b..e599258eb5453ba 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query26.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query26.groovy
@@ -28,7 +28,6 @@ suite("query26") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query27.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query27.groovy
index efe99bddc35d6f9..312b29a2056d36e 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query27.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query27.groovy
@@ -28,7 +28,6 @@ suite("query27") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query28.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query28.groovy
index 8c43bbaf31144aa..a8151135102b1c7 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query28.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query28.groovy
@@ -28,7 +28,6 @@ suite("query28") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query29.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query29.groovy
index 80e40109de3a7a7..e1116a865dd8da7 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query29.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query29.groovy
@@ -28,7 +28,6 @@ suite("query29") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query3.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query3.groovy
index 13287c729b66572..aace9720f393ffb 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query3.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query3.groovy
@@ -28,7 +28,6 @@ suite("query3") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query30.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query30.groovy
index 254bd491cca3a7e..6afc169c8ae2dc5 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query30.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query30.groovy
@@ -28,7 +28,6 @@ suite("query30") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query31.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query31.groovy
index a7058f1ce8e31a1..bb11fdd0951d774 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query31.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query31.groovy
@@ -28,7 +28,6 @@ suite("query31") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query33.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query33.groovy
index a5d40f5c3f8ea1e..96697c0d9845955 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query33.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query33.groovy
@@ -28,7 +28,6 @@ suite("query33") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query34.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query34.groovy
index e78e64a55dcc43a..c222d6b0bece023 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query34.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query34.groovy
@@ -28,7 +28,6 @@ suite("query34") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query35.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query35.groovy
index c1b8381d904c39f..f4bdeb2db8c61ce 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query35.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query35.groovy
@@ -28,7 +28,6 @@ suite("query35") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query36.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query36.groovy
index df114af301bfc82..a1310d235ec7960 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query36.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query36.groovy
@@ -28,7 +28,6 @@ suite("query36") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query37.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query37.groovy
index c7666029e42d76b..5d48768e9b866b2 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query37.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query37.groovy
@@ -28,7 +28,6 @@ suite("query37") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query39.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query39.groovy
index 10257ad486adab9..a7e321c22656f8d 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query39.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query39.groovy
@@ -28,7 +28,6 @@ suite("query39") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query4.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query4.groovy
index d0d92981c73e6a3..f03af2536173de5 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query4.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query4.groovy
@@ -28,7 +28,6 @@ suite("query4") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query40.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query40.groovy
index 1492918741fcfe6..38cfafa6560581a 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query40.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query40.groovy
@@ -28,7 +28,6 @@ suite("query40") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query41.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query41.groovy
index 8cd49d695fc938a..b9c4a9231e533c2 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query41.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query41.groovy
@@ -28,7 +28,6 @@ suite("query41") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query42.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query42.groovy
index cc4889c9f8aef5d..e4f6a9da4b376a3 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query42.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query42.groovy
@@ -28,7 +28,6 @@ suite("query42") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query43.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query43.groovy
index 135fc558dbc6cab..e4231b09aa67387 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query43.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query43.groovy
@@ -28,7 +28,6 @@ suite("query43") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query44.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query44.groovy
index b6090e61f5555d0..fc972137d59b27f 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query44.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query44.groovy
@@ -28,7 +28,6 @@ suite("query44") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query45.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query45.groovy
index 3a22a33b5953092..ee73c4d7287a413 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query45.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query45.groovy
@@ -28,7 +28,6 @@ suite("query45") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query46.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query46.groovy
index 9ab5f90b9853de6..dd33c8ae9192d72 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query46.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query46.groovy
@@ -28,7 +28,6 @@ suite("query46") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query47.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query47.groovy
index d9765e1ea255dbc..d523df26a65240f 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query47.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query47.groovy
@@ -28,7 +28,6 @@ suite("query47") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query48.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query48.groovy
index 31411adfc404ced..3a2258662d5e487 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query48.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query48.groovy
@@ -28,7 +28,6 @@ suite("query48") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query49.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query49.groovy
index 148c9f6fbfa7a2c..a250ee0abc49cc8 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query49.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query49.groovy
@@ -28,7 +28,6 @@ suite("query49") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query5.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query5.groovy
index 518bb15598b0df6..1b549f2707ae238 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query5.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query5.groovy
@@ -28,7 +28,6 @@ suite("query5") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query50.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query50.groovy
index f2e9ab38e5dd627..8ccc569590b75fb 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query50.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query50.groovy
@@ -28,7 +28,6 @@ suite("query50") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query51.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query51.groovy
index da9ede0d59a10db..a98b98ec910062d 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query51.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query51.groovy
@@ -28,7 +28,6 @@ suite("query51") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query52.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query52.groovy
index 8787c0f9e4ea978..880f3afd167272c 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query52.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query52.groovy
@@ -28,7 +28,6 @@ suite("query52") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query53.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query53.groovy
index edc3c3ae551d31e..42e7a3474ba745b 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query53.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query53.groovy
@@ -28,7 +28,6 @@ suite("query53") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query54.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query54.groovy
index 79d5cd78c09b278..6918f70d5cafa5d 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query54.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query54.groovy
@@ -28,7 +28,6 @@ suite("query54") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query55.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query55.groovy
index f7c99cff24bd52a..70fe72657865199 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query55.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query55.groovy
@@ -28,7 +28,6 @@ suite("query55") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query56.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query56.groovy
index a311cf8771405b4..2aaceceedb6e45b 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query56.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query56.groovy
@@ -28,7 +28,6 @@ suite("query56") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query57.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query57.groovy
index f29965d5a1bfc84..47ca56f0650d3ad 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query57.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query57.groovy
@@ -28,7 +28,6 @@ suite("query57") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query58.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query58.groovy
index 3589e984a9cc74d..0cb76fca23534c5 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query58.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query58.groovy
@@ -28,7 +28,6 @@ suite("query58") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query59.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query59.groovy
index 8ca3c6804350725..3b3ecfd9068fcc3 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query59.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query59.groovy
@@ -28,7 +28,6 @@ suite("query59") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query6.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query6.groovy
index ff89e5b12b621f7..b1a51aa274d4f52 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query6.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query6.groovy
@@ -28,7 +28,6 @@ suite("query6") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query60.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query60.groovy
index ddcd041e001be09..f323b96ed5683ae 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query60.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query60.groovy
@@ -28,7 +28,6 @@ suite("query60") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query61.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query61.groovy
index 6b5aa9a1120cf7c..de78bd3fe46242e 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query61.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query61.groovy
@@ -28,7 +28,6 @@ suite("query61") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query62.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query62.groovy
index 4fe6bf0419e158a..de6a16151151348 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query62.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query62.groovy
@@ -28,7 +28,6 @@ suite("query62") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query63.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query63.groovy
index e17ff538c96e27d..34b10eaefd827ad 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query63.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query63.groovy
@@ -28,7 +28,6 @@ suite("query63") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query64.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query64.groovy
index 67f825728514dfd..25f6f33ac84db6e 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query64.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query64.groovy
@@ -28,7 +28,6 @@ suite("query64") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query65.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query65.groovy
index c317dbc653f4ec7..4aeb9cb88d9c5d8 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query65.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query65.groovy
@@ -28,7 +28,6 @@ suite("query65") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query66.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query66.groovy
index c55b1e7298d8440..3ed91231c6c2a51 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query66.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query66.groovy
@@ -28,7 +28,6 @@ suite("query66") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query67.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query67.groovy
index 1e18239fea0e496..dd7d2c1bfec035a 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query67.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query67.groovy
@@ -28,7 +28,6 @@ suite("query67") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query68.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query68.groovy
index 1f1caf7a5240d5d..6e721e5eabf45d1 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query68.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query68.groovy
@@ -28,7 +28,6 @@ suite("query68") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query69.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query69.groovy
index 8037d49d7579a61..dafcb98e9265aa7 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query69.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query69.groovy
@@ -28,7 +28,6 @@ suite("query69") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query7.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query7.groovy
index 845f70ef57fedc5..af7f9913c680378 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query7.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query7.groovy
@@ -28,7 +28,6 @@ suite("query7") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query70.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query70.groovy
index 9e4d2063491316d..beab5d0277e690b 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query70.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query70.groovy
@@ -28,7 +28,6 @@ suite("query70") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query71.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query71.groovy
index 808cad733178432..d52f597c6b258d5 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query71.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query71.groovy
@@ -28,7 +28,6 @@ suite("query71") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query72.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query72.groovy
index cd66196995358cb..aa2d674cc33f16e 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query72.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query72.groovy
@@ -28,7 +28,6 @@ suite("query72") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query73.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query73.groovy
index 2d1ac97fae1da0f..bac050095a88355 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query73.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query73.groovy
@@ -28,7 +28,6 @@ suite("query73") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query74.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query74.groovy
index 944e97c1ef1f577..a0ef28422cd655e 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query74.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query74.groovy
@@ -28,7 +28,6 @@ suite("query74") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query75.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query75.groovy
index b91fbb54fa8194a..5109f9a2acbbe30 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query75.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query75.groovy
@@ -28,7 +28,6 @@ suite("query75") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query76.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query76.groovy
index c83d452b0545e65..e4ab83d22d61efc 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query76.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query76.groovy
@@ -28,7 +28,6 @@ suite("query76") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query77.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query77.groovy
index 6100f7d42a8a85a..3e5fada6bb4aefb 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query77.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query77.groovy
@@ -28,7 +28,6 @@ suite("query77") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query78.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query78.groovy
index 68987e1a80db8ce..f8186f83278da21 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query78.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query78.groovy
@@ -28,7 +28,6 @@ suite("query78") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query79.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query79.groovy
index 9b6bb2918ce49e2..2d43c4cda6db31d 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query79.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query79.groovy
@@ -28,7 +28,6 @@ suite("query79") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query8.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query8.groovy
index 004c59cb5b3076c..04481786cb68906 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query8.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query8.groovy
@@ -28,7 +28,6 @@ suite("query8") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query80.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query80.groovy
index 57fd016217d51b6..84cd9a1c1321962 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query80.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query80.groovy
@@ -28,7 +28,6 @@ suite("query80") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query81.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query81.groovy
index a4ce847b332e9e7..3e2dea3e09bdbea 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query81.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query81.groovy
@@ -28,7 +28,6 @@ suite("query81") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query82.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query82.groovy
index f0536fbc9f21901..65e93ce43d19125 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query82.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query82.groovy
@@ -28,7 +28,6 @@ suite("query82") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query83.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query83.groovy
index 11b791bb804acce..70fbc4ce0acd7e3 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query83.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query83.groovy
@@ -28,7 +28,6 @@ suite("query83") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query84.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query84.groovy
index db5bb633b429cda..399e82338c13edc 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query84.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query84.groovy
@@ -28,7 +28,6 @@ suite("query84") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query85.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query85.groovy
index 2dee4d2872b4f2f..6352b8d9bfb05db 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query85.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query85.groovy
@@ -28,7 +28,6 @@ suite("query85") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query86.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query86.groovy
index fb9ff1c39e701f1..2200b2d71b836a2 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query86.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query86.groovy
@@ -28,7 +28,6 @@ suite("query86") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query87.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query87.groovy
index 664c930f03f32b5..f7a38a963c5c75d 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query87.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query87.groovy
@@ -28,7 +28,6 @@ suite("query87") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query88.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query88.groovy
index 05d5d8f52d099a4..e0a2c83153fb683 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query88.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query88.groovy
@@ -28,7 +28,6 @@ suite("query88") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query89.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query89.groovy
index 5de381c5eaa69a7..c0ecd01ae44cb5e 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query89.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query89.groovy
@@ -28,7 +28,6 @@ suite("query89") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query9.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query9.groovy
index 63359a154eda53b..fbd7549e41034c9 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query9.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query9.groovy
@@ -28,7 +28,6 @@ suite("query9") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query90.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query90.groovy
index 25da47edf8f0a4d..26645b7924a9b22 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query90.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query90.groovy
@@ -28,7 +28,6 @@ suite("query90") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query91.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query91.groovy
index e823e71388c288b..3f72b439c1627d5 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query91.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query91.groovy
@@ -28,7 +28,6 @@ suite("query91") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query92.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query92.groovy
index 0f13b123f9e2fea..77553e938c10b80 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query92.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query92.groovy
@@ -28,7 +28,6 @@ suite("query92") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query93.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query93.groovy
index d18af249549b2e5..bb5f53a01c895bb 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query93.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query93.groovy
@@ -28,7 +28,6 @@ suite("query93") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query94.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query94.groovy
index a0ca4799a1520b6..a3bfc5466a61c44 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query94.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query94.groovy
@@ -28,7 +28,6 @@ suite("query94") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query95.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query95.groovy
index 3e32c96597d5dc3..dbb8d1424f52a72 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query95.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query95.groovy
@@ -28,7 +28,6 @@ suite("query95") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query96.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query96.groovy
index e1929776b860c6e..925d7866adc8d5e 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query96.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query96.groovy
@@ -28,7 +28,6 @@ suite("query96") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query98.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query98.groovy
index 3876cf4f1ebff3b..36b2d5cb45bc1d6 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query98.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query98.groovy
@@ -28,7 +28,6 @@ suite("query98") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query99.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query99.groovy
index 7678b172cab6699..5377c23acb8ccd6 100644
--- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query99.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf1000/shape/query99.groovy
@@ -28,7 +28,6 @@ suite("query99") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/ddl/shape.tmpl b/regression-test/suites/shape_check/tpcds_sf10t_orc/ddl/shape.tmpl
index 9a26f444e293cc2..75fc0ec7029fb12 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/ddl/shape.tmpl
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/ddl/shape.tmpl
@@ -29,7 +29,6 @@ suite("query{--}") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query1.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query1.groovy
index b09ba3749dc821f..9ff8bd914a021fc 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query1.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query1.groovy
@@ -29,7 +29,7 @@ suite("query1") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
+         set parallel_pipeline_task_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query10.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query10.groovy
index 4cbb1818b6dc9ed..89f48d4e4bb72d4 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query10.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query10.groovy
@@ -29,7 +29,6 @@ suite("query10") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query11.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query11.groovy
index 482ceb763132ab6..c0d5e2e43604b8f 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query11.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query11.groovy
@@ -29,7 +29,6 @@ suite("query11") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query12.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query12.groovy
index f9f85398b2ec2c1..6293f3625bbfad8 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query12.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query12.groovy
@@ -29,7 +29,6 @@ suite("query12") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query13.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query13.groovy
index a2f8a97e11fb002..c32d3ae22ffad42 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query13.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query13.groovy
@@ -29,7 +29,6 @@ suite("query13") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query14.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query14.groovy
index 716356b71b9010d..b24f85335836aef 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query14.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query14.groovy
@@ -29,7 +29,6 @@ suite("query14") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query15.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query15.groovy
index 08b775a0b4cd2ca..999167e888b0747 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query15.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query15.groovy
@@ -29,7 +29,6 @@ suite("query15") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query16.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query16.groovy
index 2d06cc887c559e7..cbd741bcc50dc53 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query16.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query16.groovy
@@ -29,7 +29,6 @@ suite("query16") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query17.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query17.groovy
index 36de5b71d2f3144..13b9bfae478ba80 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query17.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query17.groovy
@@ -29,7 +29,6 @@ suite("query17") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query18.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query18.groovy
index 9f06b83351db743..80c5c8407067704 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query18.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query18.groovy
@@ -29,7 +29,6 @@ suite("query18") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query19.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query19.groovy
index 0a90db8a7bb3049..e95004bcdc09dd7 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query19.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query19.groovy
@@ -29,7 +29,6 @@ suite("query19") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query2.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query2.groovy
index eafbc743cd50722..a8c08f0047dc6c9 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query2.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query2.groovy
@@ -29,7 +29,6 @@ suite("query2") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query20.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query20.groovy
index 633310f974d7c44..4c8ffba0185cd10 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query20.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query20.groovy
@@ -29,7 +29,6 @@ suite("query20") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query21.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query21.groovy
index fb2d3590c054d28..d84d307c13be42b 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query21.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query21.groovy
@@ -29,7 +29,6 @@ suite("query21") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query22.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query22.groovy
index d5b76e556249c6a..c0dace93fef2d21 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query22.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query22.groovy
@@ -29,7 +29,6 @@ suite("query22") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query23.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query23.groovy
index a21c4feef861559..fc8b068f66b603e 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query23.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query23.groovy
@@ -29,7 +29,6 @@ suite("query23") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query24.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query24.groovy
index eec85d0f8356cf7..5407176eca2bb97 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query24.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query24.groovy
@@ -29,7 +29,6 @@ suite("query24") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query25.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query25.groovy
index eae27a1e818a703..c57fa910e3e0927 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query25.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query25.groovy
@@ -29,7 +29,6 @@ suite("query25") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query26.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query26.groovy
index 6f9e1973c7d8f3f..f63e3bc5b9483f8 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query26.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query26.groovy
@@ -29,7 +29,6 @@ suite("query26") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query27.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query27.groovy
index 239e57f4eaac197..13ce7941506a326 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query27.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query27.groovy
@@ -29,7 +29,6 @@ suite("query27") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query28.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query28.groovy
index 74fcddd47057a7c..4820eda213010af 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query28.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query28.groovy
@@ -29,7 +29,6 @@ suite("query28") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query29.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query29.groovy
index ec3b30e36a169ea..0be9f444e82f41f 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query29.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query29.groovy
@@ -29,7 +29,6 @@ suite("query29") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query3.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query3.groovy
index eeced1e881f0dc7..5ea0f50546e3e53 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query3.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query3.groovy
@@ -29,7 +29,6 @@ suite("query3") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query30.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query30.groovy
index 3abaaa8862cb913..5537f86a4691c43 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query30.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query30.groovy
@@ -29,7 +29,6 @@ suite("query30") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query31.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query31.groovy
index 381484660a7d928..3013c27b38a7e40 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query31.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query31.groovy
@@ -29,7 +29,6 @@ suite("query31") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query32.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query32.groovy
index 934c962261e8997..16f161755522689 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query32.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query32.groovy
@@ -29,7 +29,6 @@ suite("query32") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query33.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query33.groovy
index 448b6332e8f7340..bc861b8a80911f2 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query33.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query33.groovy
@@ -29,7 +29,6 @@ suite("query33") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query34.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query34.groovy
index 80acc7f72e5ab29..66c5ab3db09fea6 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query34.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query34.groovy
@@ -29,7 +29,6 @@ suite("query34") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query35.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query35.groovy
index 9ba8368f3c1147b..fd0af5128f77a22 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query35.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query35.groovy
@@ -29,7 +29,6 @@ suite("query35") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query36.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query36.groovy
index b02cf7ceec4ad36..b774f339774dfd0 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query36.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query36.groovy
@@ -29,7 +29,6 @@ suite("query36") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query37.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query37.groovy
index bc891808012e289..e1bfe076fe84b22 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query37.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query37.groovy
@@ -29,7 +29,6 @@ suite("query37") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query38.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query38.groovy
index 2edb416929885ff..34cc84c20f6446e 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query38.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query38.groovy
@@ -29,7 +29,6 @@ suite("query38") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query39.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query39.groovy
index ee3d855f57361c8..58fb2786d59420b 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query39.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query39.groovy
@@ -29,7 +29,6 @@ suite("query39") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query4.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query4.groovy
index 76ab78c6274c62e..5fca219dd96ef14 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query4.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query4.groovy
@@ -29,7 +29,6 @@ suite("query4") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query40.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query40.groovy
index 862bfafcfe1d4c0..396f428537a670a 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query40.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query40.groovy
@@ -29,7 +29,6 @@ suite("query40") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query41.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query41.groovy
index f0d71886c58d155..d712d6182fb6b74 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query41.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query41.groovy
@@ -29,7 +29,6 @@ suite("query41") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query42.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query42.groovy
index f15088227fb372b..e3223d9d785ab45 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query42.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query42.groovy
@@ -29,7 +29,6 @@ suite("query42") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query43.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query43.groovy
index 9d5e94bd7fb25e7..a7ab3eb9c700a27 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query43.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query43.groovy
@@ -29,7 +29,6 @@ suite("query43") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query44.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query44.groovy
index 19416a0a2b85133..75f51b2ea0e7875 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query44.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query44.groovy
@@ -29,7 +29,6 @@ suite("query44") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query45.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query45.groovy
index b0efcedf71157d7..50bc5d61561ac2c 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query45.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query45.groovy
@@ -29,7 +29,6 @@ suite("query45") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query46.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query46.groovy
index 010ca3d08934d5e..1854f3f966dea77 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query46.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query46.groovy
@@ -29,7 +29,6 @@ suite("query46") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query47.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query47.groovy
index 68f186f6eb01762..8b831761e28af35 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query47.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query47.groovy
@@ -29,7 +29,6 @@ suite("query47") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query48.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query48.groovy
index cb5966bb03e9b7a..b51696a6084ff84 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query48.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query48.groovy
@@ -29,7 +29,6 @@ suite("query48") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query49.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query49.groovy
index 87d5898934809b0..df87a2e00c50575 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query49.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query49.groovy
@@ -29,7 +29,6 @@ suite("query49") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query5.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query5.groovy
index 7a017fe9bd55177..a327674ecadff3a 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query5.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query5.groovy
@@ -29,7 +29,6 @@ suite("query5") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query50.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query50.groovy
index bc933c42d03a238..ba4fdeaf0a31196 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query50.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query50.groovy
@@ -29,7 +29,6 @@ suite("query50") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query51.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query51.groovy
index 79d7dd575ca11f2..daf8a8a20a44328 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query51.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query51.groovy
@@ -29,7 +29,6 @@ suite("query51") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query52.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query52.groovy
index 2d4a8489f012418..e0825cc6f124d89 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query52.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query52.groovy
@@ -29,7 +29,6 @@ suite("query52") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query53.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query53.groovy
index 3c50571c57950b7..3e9426e7ae92d6d 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query53.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query53.groovy
@@ -29,7 +29,6 @@ suite("query53") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query54.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query54.groovy
index 277edbe3f33466c..ff8b80aaa58f84e 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query54.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query54.groovy
@@ -29,7 +29,6 @@ suite("query54") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query55.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query55.groovy
index 407c432af68c2d1..f9c1375f2052c66 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query55.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query55.groovy
@@ -29,7 +29,6 @@ suite("query55") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query56.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query56.groovy
index 525032cc4c662e9..2bf4c76b401cc4d 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query56.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query56.groovy
@@ -29,7 +29,6 @@ suite("query56") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query57.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query57.groovy
index 570ccdf8b625ce1..1b192c1d8f4cde7 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query57.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query57.groovy
@@ -29,7 +29,6 @@ suite("query57") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query58.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query58.groovy
index ed78699b0b66105..ee7bea6b17b297c 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query58.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query58.groovy
@@ -29,7 +29,6 @@ suite("query58") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query59.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query59.groovy
index 7c0e3126fb00953..77038c66a9d79e3 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query59.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query59.groovy
@@ -29,7 +29,6 @@ suite("query59") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query6.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query6.groovy
index e51edf7df0a8f00..3a8c4fd5b7e6899 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query6.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query6.groovy
@@ -29,7 +29,6 @@ suite("query6") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query60.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query60.groovy
index 595f8dd65a0f584..e42f6456149acc5 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query60.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query60.groovy
@@ -29,7 +29,6 @@ suite("query60") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query61.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query61.groovy
index 9078b2735829e29..e431911b098951b 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query61.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query61.groovy
@@ -29,7 +29,6 @@ suite("query61") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query62.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query62.groovy
index cf01109c7792945..a87842297faa56d 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query62.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query62.groovy
@@ -29,7 +29,6 @@ suite("query62") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query63.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query63.groovy
index 17f99583c11ee9c..4b8602e9bbc7793 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query63.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query63.groovy
@@ -29,7 +29,6 @@ suite("query63") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query64.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query64.groovy
index 82087de873ceff0..a89a4c57c439ac0 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query64.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query64.groovy
@@ -29,7 +29,6 @@ suite("query64") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query65.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query65.groovy
index 6369f02078c7008..5934a333b39fad8 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query65.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query65.groovy
@@ -29,7 +29,6 @@ suite("query65") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query66.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query66.groovy
index 450c224b70963b5..6120c2b9fe98c93 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query66.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query66.groovy
@@ -29,7 +29,6 @@ suite("query66") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query67.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query67.groovy
index 9c066e838416a69..380880ca147b22c 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query67.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query67.groovy
@@ -29,7 +29,6 @@ suite("query67") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query68.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query68.groovy
index ada357640a4b0c9..0f45e8212c3f41f 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query68.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query68.groovy
@@ -29,7 +29,6 @@ suite("query68") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query69.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query69.groovy
index 73e9ebdd0529f58..a7144c297dacb4b 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query69.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query69.groovy
@@ -29,7 +29,6 @@ suite("query69") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query7.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query7.groovy
index cb18f6341a43ca5..c138acafd94bf85 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query7.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query7.groovy
@@ -29,7 +29,6 @@ suite("query7") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query70.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query70.groovy
index 79b189284e60fa5..7d318e9aaaa88d6 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query70.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query70.groovy
@@ -29,7 +29,6 @@ suite("query70") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query71.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query71.groovy
index 6bfb2633732647b..7786310d4c2868b 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query71.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query71.groovy
@@ -29,7 +29,6 @@ suite("query71") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query72.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query72.groovy
index 9ef1f85a4d65059..76c8668db57672a 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query72.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query72.groovy
@@ -29,7 +29,6 @@ suite("query72") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query73.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query73.groovy
index 1da37f1f77c9040..6c887d21bd85b80 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query73.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query73.groovy
@@ -29,7 +29,6 @@ suite("query73") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query74.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query74.groovy
index fb06f43c8520bb1..96edd6344078cfe 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query74.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query74.groovy
@@ -29,7 +29,6 @@ suite("query74") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query75.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query75.groovy
index 8dec01afbd9f1ca..896f7257ca52c1c 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query75.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query75.groovy
@@ -29,7 +29,6 @@ suite("query75") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query76.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query76.groovy
index b9de4e0243d3272..9deeb7d73b4bbf9 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query76.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query76.groovy
@@ -29,7 +29,6 @@ suite("query76") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query77.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query77.groovy
index 7b966b5c10c17c3..cd9f99f3fd4e644 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query77.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query77.groovy
@@ -29,7 +29,6 @@ suite("query77") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query78.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query78.groovy
index 74bb2ecbaa6344c..84fafc90bc6b20e 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query78.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query78.groovy
@@ -29,7 +29,6 @@ suite("query78") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query79.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query79.groovy
index eb66d7bb3e96480..6d9d2d8b4bcf2a8 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query79.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query79.groovy
@@ -29,7 +29,6 @@ suite("query79") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query8.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query8.groovy
index 428594e4e509d41..4935ca00630def5 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query8.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query8.groovy
@@ -29,7 +29,6 @@ suite("query8") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query80.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query80.groovy
index f4d3ed5847097c9..4d61e0a114dcbef 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query80.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query80.groovy
@@ -29,7 +29,6 @@ suite("query80") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query81.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query81.groovy
index 05a51faffa8ffcc..bb334d50ad72e55 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query81.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query81.groovy
@@ -29,7 +29,6 @@ suite("query81") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query82.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query82.groovy
index 6a17545e339bde5..d3864b17ebd4e4e 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query82.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query82.groovy
@@ -29,7 +29,6 @@ suite("query82") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query83.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query83.groovy
index 37565aa1cbca98e..b42e4aa3f260b7c 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query83.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query83.groovy
@@ -29,7 +29,6 @@ suite("query83") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query84.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query84.groovy
index 33e97e1c2571140..3193427e25f2a6f 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query84.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query84.groovy
@@ -29,7 +29,6 @@ suite("query84") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query85.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query85.groovy
index d7c71bd0a327019..eee86419b6246e1 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query85.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query85.groovy
@@ -29,7 +29,6 @@ suite("query85") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query86.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query86.groovy
index 0d6d1d88d7cbdb7..181c420876f9b4f 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query86.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query86.groovy
@@ -29,7 +29,6 @@ suite("query86") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query87.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query87.groovy
index 7e9d529b96a2035..1a520db128aa6d9 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query87.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query87.groovy
@@ -29,7 +29,6 @@ suite("query87") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query88.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query88.groovy
index e5b0f1e665f729d..2fc0f8202a2b565 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query88.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query88.groovy
@@ -29,7 +29,6 @@ suite("query88") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query89.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query89.groovy
index bbd6a98205bfac9..4d59bbb1576c7e9 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query89.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query89.groovy
@@ -29,7 +29,6 @@ suite("query89") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query9.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query9.groovy
index 026e327d4b97131..54b71b5cb6e2d97 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query9.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query9.groovy
@@ -29,7 +29,6 @@ suite("query9") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query90.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query90.groovy
index d23f0714d0da975..a46ef24a9a6bb90 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query90.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query90.groovy
@@ -29,7 +29,6 @@ suite("query90") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query91.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query91.groovy
index bad3d75039816da..036e70bfd3a4c62 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query91.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query91.groovy
@@ -29,7 +29,6 @@ suite("query91") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query92.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query92.groovy
index 56e470d16523949..2ad6b9d3dfc460e 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query92.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query92.groovy
@@ -29,7 +29,6 @@ suite("query92") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query93.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query93.groovy
index a1074c8ea52cdd7..c01fe8f53686ce1 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query93.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query93.groovy
@@ -29,7 +29,6 @@ suite("query93") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query94.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query94.groovy
index 48ff5e3da213e44..476dcd5416be2cf 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query94.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query94.groovy
@@ -29,7 +29,6 @@ suite("query94") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query95.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query95.groovy
index 78c4d0e81df3dd2..4aba7457d04c37f 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query95.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query95.groovy
@@ -29,7 +29,6 @@ suite("query95") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query96.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query96.groovy
index 148908951723b30..05d5621cbfcf56f 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query96.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query96.groovy
@@ -29,7 +29,6 @@ suite("query96") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query97.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query97.groovy
index 10dd934df3fa0a5..e99aa0ffe9b1fd8 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query97.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query97.groovy
@@ -29,7 +29,6 @@ suite("query97") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query98.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query98.groovy
index be1b13c320a7bd5..9a3ecdf700f99a6 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query98.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query98.groovy
@@ -29,7 +29,6 @@ suite("query98") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query99.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query99.groovy
index fe6a2ea601ca612..0395a07ec8ba640 100644
--- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query99.groovy
+++ b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query99.groovy
@@ -29,7 +29,6 @@ suite("query99") {
          set enable_fallback_to_original_planner=false;
          set exec_mem_limit=21G;
          set be_number_for_test=3;
-         set parallel_fragment_exec_instance_num=8; ;
          set parallel_pipeline_task_num=8;
          set forbid_unknown_col_stats=true;
          set enable_nereids_timeout = false;
diff --git a/regression-test/suites/shape_check/tpch_sf1000/runtime_filter/test_pushdown_setop.groovy b/regression-test/suites/shape_check/tpch_sf1000/runtime_filter/test_pushdown_setop.groovy
index 97edccd1bc3c686..ebd96da5905c5ce 100644
--- a/regression-test/suites/shape_check/tpch_sf1000/runtime_filter/test_pushdown_setop.groovy
+++ b/regression-test/suites/shape_check/tpch_sf1000/runtime_filter/test_pushdown_setop.groovy
@@ -28,7 +28,6 @@ suite("test_pushdown_setop") {
     sql 'set enable_fallback_to_original_planner=false'
     sql 'set exec_mem_limit=21G'
     sql 'set be_number_for_test=3'
-    sql 'set parallel_fragment_exec_instance_num=8; '
     sql 'set parallel_pipeline_task_num=8; '
     sql 'set forbid_unknown_col_stats=true'
     sql 'set enable_nereids_timeout = false'
diff --git a/regression-test/suites/ssb_sf100_p2/sql/bigbuild.sql b/regression-test/suites/ssb_sf100_p2/sql/bigbuild.sql
index 2cadd8ed37655ae..bb5b72bbddbae25 100644
--- a/regression-test/suites/ssb_sf100_p2/sql/bigbuild.sql
+++ b/regression-test/suites/ssb_sf100_p2/sql/bigbuild.sql
@@ -15,4 +15,4 @@
 -- specific language governing permissions and limitations
 -- under the License.
 
-select  /*+SET_VAR(parallel_fragment_exec_instance_num=1,parallel_pipeline_task_num=1)*/ count(*) from lineorder a left join lineorder b on a.lo_orderkey=b.lo_orderkey;
\ No newline at end of file
+select  /*+SET_VAR(parallel_pipeline_task_num=1)*/ count(*) from lineorder a left join lineorder b on a.lo_orderkey=b.lo_orderkey;
\ No newline at end of file
diff --git a/regression-test/suites/ssb_sf1_p2/nereids/q1.1.groovy b/regression-test/suites/ssb_sf1_p2/nereids/q1.1.groovy
index a24de2297df6c41..ff02314923b9566 100644
--- a/regression-test/suites/ssb_sf1_p2/nereids/q1.1.groovy
+++ b/regression-test/suites/ssb_sf1_p2/nereids/q1.1.groovy
@@ -28,7 +28,7 @@ suite("ssb_sf1_q1_1_nereids") {
 
     test {
         // sql(new File(context.file.parentFile, "../sql/q1.1.sql").text)
-        sql """SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1)*/  
+        sql """SELECT /*+SET_VAR(parallel_pipeline_task_num=1)*/
         SUM(lo_extendedprice*lo_discount) AS
         REVENUE
         FROM  lineorder, date
diff --git a/regression-test/suites/ssb_sf1_p2/nereids/q1.2.groovy b/regression-test/suites/ssb_sf1_p2/nereids/q1.2.groovy
index ca73f3a4e4ca4fc..46d3f8b13987a42 100644
--- a/regression-test/suites/ssb_sf1_p2/nereids/q1.2.groovy
+++ b/regression-test/suites/ssb_sf1_p2/nereids/q1.2.groovy
@@ -30,7 +30,7 @@ suite("ssb_sf1_q1_2_nereids") {
     test {
         // sql(new File(context.file.parentFile, "../sql/q1.2.sql").text)
         sql """
-        SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1)*/  
+        SELECT /*+SET_VAR(parallel_pipeline_task_num=1)*/
         SUM(lo_extendedprice*lo_discount) AS
         REVENUE
         FROM  lineorder, date
diff --git a/regression-test/suites/ssb_sf1_p2/nereids/q1.3.groovy b/regression-test/suites/ssb_sf1_p2/nereids/q1.3.groovy
index b0af2bed32988b0..2212cd4603f665d 100644
--- a/regression-test/suites/ssb_sf1_p2/nereids/q1.3.groovy
+++ b/regression-test/suites/ssb_sf1_p2/nereids/q1.3.groovy
@@ -29,7 +29,7 @@ suite("ssb_sf1_q1_3_nereids") {
 
     test {
         // sql(new File(context.file.parentFile, "../sql/q1.3.sql").text)
-        sql """SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1)*/  
+        sql """SELECT /*+SET_VAR(parallel_pipeline_task_num=1)*/
         SUM(lo_extendedprice*lo_discount) AS
         REVENUE
         FROM  lineorder, date
diff --git a/regression-test/suites/ssb_sf1_p2/nereids/q2.1.groovy b/regression-test/suites/ssb_sf1_p2/nereids/q2.1.groovy
index dd76b45cb8110bf..e2fd37fa5618c69 100644
--- a/regression-test/suites/ssb_sf1_p2/nereids/q2.1.groovy
+++ b/regression-test/suites/ssb_sf1_p2/nereids/q2.1.groovy
@@ -29,7 +29,7 @@ suite("ssb_sf1_q2_1_nereids") {
 
     test {
         // sql(new File(context.file.parentFile, "../sql/q2.1.sql").text)
-        sql """SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1)*/  
+        sql """SELECT /*+SET_VAR(parallel_pipeline_task_num=1)*/
         SUM(lo_revenue), d_year, p_brand
         FROM lineorder, date, part, supplier
         WHERE lo_orderdate = d_datekey
diff --git a/regression-test/suites/ssb_sf1_p2/nereids/q2.2.groovy b/regression-test/suites/ssb_sf1_p2/nereids/q2.2.groovy
index 712744acb67b692..9a20ea288b0b55e 100644
--- a/regression-test/suites/ssb_sf1_p2/nereids/q2.2.groovy
+++ b/regression-test/suites/ssb_sf1_p2/nereids/q2.2.groovy
@@ -29,7 +29,7 @@ suite("ssb_sf1_q2_2_nereids") {
 
     test {
         // sql(new File(context.file.parentFile, "../sql/q2.2.sql").text)
-        sql """SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1)*/  
+        sql """SELECT /*+SET_VAR(parallel_pipeline_task_num=1)*/
         SUM(lo_revenue), d_year, p_brand
         FROM lineorder, date, part, supplier
         WHERE lo_orderdate = d_datekey
diff --git a/regression-test/suites/ssb_sf1_p2/nereids/q2.3.groovy b/regression-test/suites/ssb_sf1_p2/nereids/q2.3.groovy
index e72a28730f83b78..2b0311155c1d0dc 100644
--- a/regression-test/suites/ssb_sf1_p2/nereids/q2.3.groovy
+++ b/regression-test/suites/ssb_sf1_p2/nereids/q2.3.groovy
@@ -28,7 +28,7 @@ suite("ssb_sf1_q2_3_nereids") {
 
     test {
         // sql(new File(context.file.parentFile, "../sql/q2.3.sql").text)
-        sql """SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1)*/  
+        sql """SELECT /*+SET_VAR(parallel_pipeline_task_num=1)*/
         SUM(lo_revenue), d_year, p_brand
         FROM lineorder, date, part, supplier
         WHERE  lo_orderdate = d_datekey
diff --git a/regression-test/suites/ssb_sf1_p2/nereids/q3.1.groovy b/regression-test/suites/ssb_sf1_p2/nereids/q3.1.groovy
index a9ec4fa77e0bf8c..ea05b9dacdde4b5 100644
--- a/regression-test/suites/ssb_sf1_p2/nereids/q3.1.groovy
+++ b/regression-test/suites/ssb_sf1_p2/nereids/q3.1.groovy
@@ -28,7 +28,7 @@ suite("ssb_sf1_q3_1_nereids") {
 
     test {
         // sql(new File(context.file.parentFile, "../sql/q3.1.sql").text)
-        sql """SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1)*/   
+        sql """SELECT /*+SET_VAR(parallel_pipeline_task_num=1)*/
         c_nation, s_nation, d_year,
         SUM(lo_revenue)  AS  REVENUE
         FROM customer, lineorder, supplier, date
diff --git a/regression-test/suites/ssb_sf1_p2/nereids/q3.2.groovy b/regression-test/suites/ssb_sf1_p2/nereids/q3.2.groovy
index 34687e3bbbff89a..dff174a0c60853b 100644
--- a/regression-test/suites/ssb_sf1_p2/nereids/q3.2.groovy
+++ b/regression-test/suites/ssb_sf1_p2/nereids/q3.2.groovy
@@ -28,7 +28,7 @@ suite("ssb_sf1_q3_2_nereids") {
 
     test {
         // sql(new File(context.file.parentFile, "../sql/q3.2.sql").text)
-        sql """SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1)*/  
+        sql """SELECT /*+SET_VAR(parallel_pipeline_task_num=1)*/
         c_city, s_city, d_year, sum(lo_revenue)
         AS  REVENUE
         FROM customer, lineorder, supplier, date
diff --git a/regression-test/suites/ssb_sf1_p2/nereids/q3.3.groovy b/regression-test/suites/ssb_sf1_p2/nereids/q3.3.groovy
index b7edd51750fa237..516175af08615b7 100644
--- a/regression-test/suites/ssb_sf1_p2/nereids/q3.3.groovy
+++ b/regression-test/suites/ssb_sf1_p2/nereids/q3.3.groovy
@@ -28,7 +28,7 @@ suite("ssb_sf1_q3_3_nereids") {
 
     test {
         // sql(new File(context.file.parentFile, "../sql/q3.3.sql").text)
-        sql """SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1)*/   
+        sql """SELECT /*+SET_VAR(parallel_pipeline_task_num=1)*/
         c_city, s_city, d_year, SUM(lo_revenue)
         AS  REVENUE
         FROM customer, lineorder, supplier, date
diff --git a/regression-test/suites/ssb_sf1_p2/nereids/q3.4.groovy b/regression-test/suites/ssb_sf1_p2/nereids/q3.4.groovy
index dd685db92edbb16..816dcea714ecbf7 100644
--- a/regression-test/suites/ssb_sf1_p2/nereids/q3.4.groovy
+++ b/regression-test/suites/ssb_sf1_p2/nereids/q3.4.groovy
@@ -28,7 +28,7 @@ suite("ssb_sf1_q3_4_nereids") {
 
     test {
         // sql(new File(context.file.parentFile, "../sql/q3.4.sql").text)
-        sql """SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1)*/  
+        sql """SELECT /*+SET_VAR(parallel_pipeline_task_num=1)*/
         c_city, s_city, d_year, SUM(lo_revenue)
         AS  REVENUE
         FROM customer, lineorder, supplier, date
diff --git a/regression-test/suites/ssb_sf1_p2/nereids/q4.1.groovy b/regression-test/suites/ssb_sf1_p2/nereids/q4.1.groovy
index e88cba1a35d8389..8656a5259aaabd0 100644
--- a/regression-test/suites/ssb_sf1_p2/nereids/q4.1.groovy
+++ b/regression-test/suites/ssb_sf1_p2/nereids/q4.1.groovy
@@ -29,7 +29,7 @@ suite("ssb_sf1_q4_1_nereids") {
 
     test {
         // sql(new File(context.file.parentFile, "../sql/q4.1.sql").text)
-        sql """SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1)*/
+        sql """SELECT /*+SET_VAR(parallel_pipeline_task_num=1)*/
         d_year, c_nation,
         SUM(lo_revenue - lo_supplycost) AS PROFIT
         FROM date, customer, supplier, part, lineorder
diff --git a/regression-test/suites/ssb_sf1_p2/nereids/q4.2.groovy b/regression-test/suites/ssb_sf1_p2/nereids/q4.2.groovy
index d4cc3c96c03b7d5..9fa59565a95c30c 100644
--- a/regression-test/suites/ssb_sf1_p2/nereids/q4.2.groovy
+++ b/regression-test/suites/ssb_sf1_p2/nereids/q4.2.groovy
@@ -28,7 +28,7 @@ suite("ssb_sf1_q4_2_nereids") {
 
     test {
         // sql(new File(context.file.parentFile, "../sql/q4.2.sql").text)
-        sql """SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1)*/
+        sql """SELECT /*+SET_VAR(parallel_pipeline_task_num=1)*/
         d_year, s_nation, p_category,
         SUM(lo_revenue - lo_supplycost) AS PROFIT
         FROM date, customer, supplier, part, lineorder
diff --git a/regression-test/suites/ssb_sf1_p2/nereids/q4.3.groovy b/regression-test/suites/ssb_sf1_p2/nereids/q4.3.groovy
index e01a5090a27ea3d..f98a8cb41a27266 100644
--- a/regression-test/suites/ssb_sf1_p2/nereids/q4.3.groovy
+++ b/regression-test/suites/ssb_sf1_p2/nereids/q4.3.groovy
@@ -28,7 +28,7 @@ suite("ssb_sf1_q4_3_nereids") {
 
     test {
         // sql(new File(context.file.parentFile, "../sql/q4.3.sql").text)
-        sql """SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=1)*/
+        sql """SELECT /*+SET_VAR(parallel_pipeline_task_num=1)*/
         d_year, s_city, p_brand,
         SUM(lo_revenue - lo_supplycost) AS PROFIT
         FROM date, customer, supplier, part, lineorder
diff --git a/regression-test/suites/tpcds_sf1_p1/functions_test/test_right_join.groovy b/regression-test/suites/tpcds_sf1_p1/functions_test/test_right_join.groovy
index 7ceabbd94768bab..ba8de551a47dff4 100644
--- a/regression-test/suites/tpcds_sf1_p1/functions_test/test_right_join.groovy
+++ b/regression-test/suites/tpcds_sf1_p1/functions_test/test_right_join.groovy
@@ -17,7 +17,7 @@
 
 suite("test_right_join") {
     sql """set disable_join_reorder=true"""
-    sql """set parallel_fragment_exec_instance_num=1"""
+    sql """set parallel_pipeline_task_num=1"""
     sql "use regression_test_tpcds_sf1_p1"
 
     qt_sql_right_outer_join_with_other_conjuncts """
diff --git a/regression-test/suites/tpch_sf1_p2/nereids/q1.groovy b/regression-test/suites/tpch_sf1_p2/nereids/q1.groovy
index d97b89687c654cd..371f693ba518fc8 100644
--- a/regression-test/suites/tpch_sf1_p2/nereids/q1.groovy
+++ b/regression-test/suites/tpch_sf1_p2/nereids/q1.groovy
@@ -53,7 +53,7 @@ suite("tpch_sf1_q1_nereids") {
     """
 
     qt_select """
-    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=16, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=false, enable_projection=false) */
+    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=16, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=false, enable_projection=false) */
         l_returnflag,
         l_linestatus,
         sum(l_quantity) as sum_qty,
diff --git a/regression-test/suites/tpch_sf1_p2/nereids/q10.groovy b/regression-test/suites/tpch_sf1_p2/nereids/q10.groovy
index 343e1eeb8bf2541..8777e642ddd4133 100644
--- a/regression-test/suites/tpch_sf1_p2/nereids/q10.groovy
+++ b/regression-test/suites/tpch_sf1_p2/nereids/q10.groovy
@@ -64,7 +64,7 @@ suite("tpch_sf1_q10_nereids") {
     """
 
     qt_select """
-    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=16, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=false, enable_projection=true) */
+    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=16, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=false, enable_projection=true) */
         c_custkey,
         c_name,
         sum(t1.l_extendedprice * (1 - t1.l_discount)) as revenue,
diff --git a/regression-test/suites/tpch_sf1_p2/nereids/q11.groovy b/regression-test/suites/tpch_sf1_p2/nereids/q11.groovy
index b8fdb1f73278a2c..75e9db44d478874 100644
--- a/regression-test/suites/tpch_sf1_p2/nereids/q11.groovy
+++ b/regression-test/suites/tpch_sf1_p2/nereids/q11.groovy
@@ -59,7 +59,7 @@ suite("tpch_sf1_q11_nereids") {
     """
 
     qt_select """
-    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=2, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=true, enable_projection=true) */
+    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=2, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=true, enable_projection=true) */
         ps_partkey,
         sum(ps_supplycost * ps_availqty) as value
     from
diff --git a/regression-test/suites/tpch_sf1_p2/nereids/q12.groovy b/regression-test/suites/tpch_sf1_p2/nereids/q12.groovy
index 20ee1d6226c0272..b533f376f6532f6 100644
--- a/regression-test/suites/tpch_sf1_p2/nereids/q12.groovy
+++ b/regression-test/suites/tpch_sf1_p2/nereids/q12.groovy
@@ -60,7 +60,7 @@ suite("tpch_sf1_q12_nereids") {
     """
 
     qt_select """
-    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=2, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=true, enable_projection=true) */
+    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=2, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=true, enable_projection=true) */
         l_shipmode,
         sum(case
             when o_orderpriority = '1-URGENT'
diff --git a/regression-test/suites/tpch_sf1_p2/nereids/q13.groovy b/regression-test/suites/tpch_sf1_p2/nereids/q13.groovy
index bc6ccb681f1a481..380147c6e25741b 100644
--- a/regression-test/suites/tpch_sf1_p2/nereids/q13.groovy
+++ b/regression-test/suites/tpch_sf1_p2/nereids/q13.groovy
@@ -52,7 +52,7 @@ suite("tpch_sf1_q13_nereids") {
         """
 
         qt_select """
-    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=16, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=true, enable_projection=true) */
+    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=16, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=true, enable_projection=true) */
         c_count,
         count(*) as custdist
     from
diff --git a/regression-test/suites/tpch_sf1_p2/nereids/q14.groovy b/regression-test/suites/tpch_sf1_p2/nereids/q14.groovy
index ea33b51d73f272c..d227cbcdbac4df1 100644
--- a/regression-test/suites/tpch_sf1_p2/nereids/q14.groovy
+++ b/regression-test/suites/tpch_sf1_p2/nereids/q14.groovy
@@ -45,7 +45,7 @@ where
     """
 
     qt_select """
-select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=8, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=true, enable_projection=true) */
+select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=8, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=true, enable_projection=true) */
     100.00 * sum(case
         when p_type like 'PROMO%'
             then l_extendedprice * (1 - l_discount)
diff --git a/regression-test/suites/tpch_sf1_p2/nereids/q15.groovy b/regression-test/suites/tpch_sf1_p2/nereids/q15.groovy
index 547091e053061cf..b18709a358204d4 100644
--- a/regression-test/suites/tpch_sf1_p2/nereids/q15.groovy
+++ b/regression-test/suites/tpch_sf1_p2/nereids/q15.groovy
@@ -71,7 +71,7 @@ suite("tpch_sf1_q15_nereids") {
     """
 
     qt_select """
-    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=8, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=true, enable_projection=true) */
+    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=8, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=true, enable_projection=true) */
         s_suppkey,
         s_name,
         s_address,
diff --git a/regression-test/suites/tpch_sf1_p2/nereids/q16.groovy b/regression-test/suites/tpch_sf1_p2/nereids/q16.groovy
index 426c687ace94623..6d4a2094c852553 100644
--- a/regression-test/suites/tpch_sf1_p2/nereids/q16.groovy
+++ b/regression-test/suites/tpch_sf1_p2/nereids/q16.groovy
@@ -62,7 +62,7 @@ order by
     """
 
     qt_select """
-select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=8, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=true, enable_projection=true) */
+select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=8, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=true, enable_projection=true) */
     p_brand,
     p_type,
     p_size,
diff --git a/regression-test/suites/tpch_sf1_p2/nereids/q17.groovy b/regression-test/suites/tpch_sf1_p2/nereids/q17.groovy
index 43f489aa2f858a0..39df0c5bb7f650c 100644
--- a/regression-test/suites/tpch_sf1_p2/nereids/q17.groovy
+++ b/regression-test/suites/tpch_sf1_p2/nereids/q17.groovy
@@ -49,7 +49,7 @@ suite("tpch_sf1_q17_nereids") {
     """
 
     qt_select """
-    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=1, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=true, enable_projection=true) */
+    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=1, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=true, enable_projection=true) */
         sum(l_extendedprice) / 7.0 as avg_yearly
     from
         lineitem join
diff --git a/regression-test/suites/tpch_sf1_p2/nereids/q18.groovy b/regression-test/suites/tpch_sf1_p2/nereids/q18.groovy
index 783aa72b1732cfc..596ed00a6a3e73e 100644
--- a/regression-test/suites/tpch_sf1_p2/nereids/q18.groovy
+++ b/regression-test/suites/tpch_sf1_p2/nereids/q18.groovy
@@ -65,7 +65,7 @@ limit 100;
     """
 
      qt_select """
-select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=16, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=true, enable_projection=true) */
+select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=16, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=true, enable_projection=true) */
     c_name,
     c_custkey,
     t3.o_orderkey,
diff --git a/regression-test/suites/tpch_sf1_p2/nereids/q19.groovy b/regression-test/suites/tpch_sf1_p2/nereids/q19.groovy
index 7c4202b8360c9e8..ed46bedb026544a 100644
--- a/regression-test/suites/tpch_sf1_p2/nereids/q19.groovy
+++ b/regression-test/suites/tpch_sf1_p2/nereids/q19.groovy
@@ -67,7 +67,7 @@ where
     """
 
     qt_select """
-select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=2, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=false, enable_projection=true) */
+select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=2, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=false, enable_projection=true) */
     sum(l_extendedprice* (1 - l_discount)) as revenue
 from
     lineitem,
diff --git a/regression-test/suites/tpch_sf1_p2/nereids/q2.groovy b/regression-test/suites/tpch_sf1_p2/nereids/q2.groovy
index dc15578ebee10ae..991bee365ba0f18 100644
--- a/regression-test/suites/tpch_sf1_p2/nereids/q2.groovy
+++ b/regression-test/suites/tpch_sf1_p2/nereids/q2.groovy
@@ -76,7 +76,7 @@ limit 100;
     """
 
     qt_select """
-        select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=1, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=false, enable_projection=true) */
+        select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=1, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=false, enable_projection=true) */
     s_acctbal,
     s_name,
     n_name,
diff --git a/regression-test/suites/tpch_sf1_p2/nereids/q20.groovy b/regression-test/suites/tpch_sf1_p2/nereids/q20.groovy
index 88e02d5cbc9ab57..73ab6effef6936a 100644
--- a/regression-test/suites/tpch_sf1_p2/nereids/q20.groovy
+++ b/regression-test/suites/tpch_sf1_p2/nereids/q20.groovy
@@ -69,7 +69,7 @@ order by
     """
 
     qt_select """
-select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=16, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=true, enable_projection=true) */
+select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=16, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=true, enable_projection=true) */
 s_name, s_address from
 supplier left semi join
 (
diff --git a/regression-test/suites/tpch_sf1_p2/nereids/q21.groovy b/regression-test/suites/tpch_sf1_p2/nereids/q21.groovy
index 7f5d6fd97afb9c4..4a8980055964d81 100644
--- a/regression-test/suites/tpch_sf1_p2/nereids/q21.groovy
+++ b/regression-test/suites/tpch_sf1_p2/nereids/q21.groovy
@@ -72,7 +72,7 @@ limit 100;
     """
 
     qt_select """
-select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=16, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=true, enable_projection=true) */
+select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=16, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=true, enable_projection=true) */
 s_name, count(*) as numwait
 from orders join
 (
diff --git a/regression-test/suites/tpch_sf1_p2/nereids/q22.groovy b/regression-test/suites/tpch_sf1_p2/nereids/q22.groovy
index 925da82cceac81f..ad8573303853357 100644
--- a/regression-test/suites/tpch_sf1_p2/nereids/q22.groovy
+++ b/regression-test/suites/tpch_sf1_p2/nereids/q22.groovy
@@ -69,7 +69,7 @@ order by
     """
 
     qt_select """
-select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=16, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=true, enable_projection=true) */
+select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=16, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=true, enable_projection=true) */
     cntrycode,
     count(*) as numcust,
     sum(c_acctbal) as totacctbal
diff --git a/regression-test/suites/tpch_sf1_p2/nereids/q3.groovy b/regression-test/suites/tpch_sf1_p2/nereids/q3.groovy
index 7079e324e91202a..f20cab9184aab46 100644
--- a/regression-test/suites/tpch_sf1_p2/nereids/q3.groovy
+++ b/regression-test/suites/tpch_sf1_p2/nereids/q3.groovy
@@ -54,7 +54,7 @@ suite("tpch_sf1_q3_nereids") {
     """
 
     qt_select """
-    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=8, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=false, enable_projection=true) */
+    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=8, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=false, enable_projection=true) */
         l_orderkey,
         sum(l_extendedprice * (1 - l_discount)) as revenue,
         o_orderdate,
diff --git a/regression-test/suites/tpch_sf1_p2/nereids/q4.groovy b/regression-test/suites/tpch_sf1_p2/nereids/q4.groovy
index 36fee9b1a3802f4..cd4dbd04e802709 100644
--- a/regression-test/suites/tpch_sf1_p2/nereids/q4.groovy
+++ b/regression-test/suites/tpch_sf1_p2/nereids/q4.groovy
@@ -53,7 +53,7 @@ suite("tpch_sf1_q4_nereids") {
     """
 
     qt_select """
-    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=4, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=false, enable_projection=true) */
+    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=4, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=false, enable_projection=true) */
         o_orderpriority,
         count(*) as order_count
     from
diff --git a/regression-test/suites/tpch_sf1_p2/nereids/q6.groovy b/regression-test/suites/tpch_sf1_p2/nereids/q6.groovy
index 3cd87d87199b983..79316069cfbefe3 100644
--- a/regression-test/suites/tpch_sf1_p2/nereids/q6.groovy
+++ b/regression-test/suites/tpch_sf1_p2/nereids/q6.groovy
@@ -41,7 +41,7 @@ suite("tpch_sf1_q6_nereids") {
     """
 
     qt_select """
-    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=1, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=false, enable_projection=true) */
+    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=1, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=false, enable_projection=true) */
         sum(l_extendedprice * l_discount) as revenue
     from
         lineitem
diff --git a/regression-test/suites/tpch_sf1_p2/nereids/q7.groovy b/regression-test/suites/tpch_sf1_p2/nereids/q7.groovy
index f38bb0937730fa8..439be2fcd030907 100644
--- a/regression-test/suites/tpch_sf1_p2/nereids/q7.groovy
+++ b/regression-test/suites/tpch_sf1_p2/nereids/q7.groovy
@@ -71,7 +71,7 @@ suite("tpch_sf1_q7_nereids") {
     """
 
     qt_select """
-    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=8, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=false, enable_projection=true) */
+    select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=8, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=false, enable_projection=true) */
         supp_nation,
         cust_nation,
         l_year,
diff --git a/regression-test/suites/tpch_sf1_p2/nereids/q8.groovy b/regression-test/suites/tpch_sf1_p2/nereids/q8.groovy
index 705fb966dc2207c..55d880402bc0ef7 100644
--- a/regression-test/suites/tpch_sf1_p2/nereids/q8.groovy
+++ b/regression-test/suites/tpch_sf1_p2/nereids/q8.groovy
@@ -70,7 +70,7 @@ order by
 
     qt_select """
 
-select /*+SET_VAR(exec_mem_limit=8589934592, parallel_fragment_exec_instance_num=8, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=false, enable_projection=true) */
+select /*+SET_VAR(exec_mem_limit=8589934592, parallel_pipeline_task_num=8, batch_size=4096, disable_join_reorder=true, enable_cost_based_join_reorder=false, enable_projection=true) */
     o_year,
     sum(case
         when nation = 'BRAZIL' then volume
diff --git a/regression-test/suites/tpch_sf1_p2/nereids/q9.groovy b/regression-test/suites/tpch_sf1_p2/nereids/q9.groovy
index d9a17eb4225880a..15376f528590eaf 100644
--- a/regression-test/suites/tpch_sf1_p2/nereids/q9.groovy
+++ b/regression-test/suites/tpch_sf1_p2/nereids/q9.groovy
@@ -64,7 +64,7 @@ suite("tpch_sf1_q9_nereids") {
     """
 
     qt_select """
-    select/*+SET_VAR(exec_mem_limit=17179869184, parallel_fragment_exec_instance_num=4, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=false, enable_projection=true, enable_runtime_filter_prune=true, runtime_filter_wait_time_ms=10000) */
+    select/*+SET_VAR(exec_mem_limit=17179869184, parallel_pipeline_task_num=4, batch_size=4096, disable_join_reorder=false, enable_cost_based_join_reorder=false, enable_projection=true, enable_runtime_filter_prune=true, runtime_filter_wait_time_ms=10000) */
         nation,
         o_year,
         sum(amount) as sum_profit
diff --git a/regression-test/suites/variant_p0/load.groovy b/regression-test/suites/variant_p0/load.groovy
index b679a3ad9d96579..8731e0459f69a9a 100644
--- a/regression-test/suites/variant_p0/load.groovy
+++ b/regression-test/suites/variant_p0/load.groovy
@@ -290,7 +290,7 @@ suite("regression_test_variant", "p0"){
         sql """insert into ${table_name} values (5, '{"i" : 1}'), (1, '{"a" : 1}')"""
         sql """insert into ${table_name} values (6, '{"j" : 1}'), (1, '{"a" : 1}')"""
         sql """insert into ${table_name} values (6, '{"k" : 1}'), (1, '{"a" : 1}')"""
-        sql "select /*+SET_VAR(batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=1,parallel_pipeline_task_num=4,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=true,parallel_scan_max_scanners_count=16,parallel_scan_min_rows_per_scanner=128,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=2,enable_parallel_result_sink=false,sort_phase_num=0,enable_nereids_planner=true,rewrite_or_to_in_predicate_threshold=2,enable_function_pushdown=true,enable_common_expr_pushdown=true,enable_local_exchange=true,partition_pruning_expand_threshold=10,enable_share_hash_table_for_broadcast_join=true,enable_two_phase_read_opt=true,enable_common_expr_pushdown_for_inverted_index=false,enable_delete_sub_predicate_v2=true,min_revocable_mem=33554432,fetch_remote_schema_timeout_seconds=120,max_fetch_remote_schema_tablet_count=512,enable_join_spill=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5) */ * from ${table_name}"
+        sql "select /*+SET_VAR(batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_pipeline_task_num=4,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=true,parallel_scan_max_scanners_count=16,parallel_scan_min_rows_per_scanner=128,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=2,enable_parallel_result_sink=false,sort_phase_num=0,enable_nereids_planner=true,rewrite_or_to_in_predicate_threshold=2,enable_function_pushdown=true,enable_common_expr_pushdown=true,enable_local_exchange=true,partition_pruning_expand_threshold=10,enable_share_hash_table_for_broadcast_join=true,enable_two_phase_read_opt=true,enable_common_expr_pushdown_for_inverted_index=false,enable_delete_sub_predicate_v2=true,min_revocable_mem=33554432,fetch_remote_schema_timeout_seconds=120,max_fetch_remote_schema_tablet_count=512,enable_join_spill=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5) */ * from ${table_name}"
         qt_sql_36_1 "select cast(v['a'] as int), cast(v['b'] as int), cast(v['c'] as int) from ${table_name} order by k limit 10"
         sql "DELETE FROM ${table_name} WHERE k=1"
         sql "select * from ${table_name}"
diff --git a/regression-test/suites/variant_p0/nested.groovy b/regression-test/suites/variant_p0/nested.groovy
index 8eaec744b029eb3..c20b5a757b2c7f1 100644
--- a/regression-test/suites/variant_p0/nested.groovy
+++ b/regression-test/suites/variant_p0/nested.groovy
@@ -87,7 +87,7 @@ suite("regression_test_variant_nested", "p0"){
         trigger_and_wait_compaction("var_nested", "full")
 
         qt_sql """select  /*+SET_VAR(batch_size=1024,broker_load_batch_size=16352,disable_streaming_preaggregations=true,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_
-parallel_pipeline_task_num=7,parallel_fragment_exec_instance_num=4,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=false,parallel_scan_max_scanners_count=16
+parallel_pipeline_task_num=7,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=false,parallel_scan_max_scanners_count=16
 ,parallel_scan_min_rows_per_scanner=128,enable_fold_constant_by_be=false,enable_rewrite_element_at_to_slot=true,runtime_filter_type=2,enable_parallel_result_sink=true,sort_phase_num=0,enable_nereids_planner=true,rewrite_or_to_in_predicate_threshold=2,enable_function_pushdown=true,enable_common_expr_pushdown=false,enable_local_exchange=true,partition_pruning_expand_threshold=10,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_common_expr_pushdown_for_inverted_index=true,enable_delete_sub_predicate_v2=true,min_revocable_mem=1,fetch_remote_schema_timeout_seconds=120,max_fetch_remote_schema_tablet_count=512,enable_join_spill=true,enable_sort_spill=true,enable_agg_spill=true,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5) */ * from var_nested where v['k2'] = 'some' order by k limit 10"""
         qt_sql """select * from var_nested where v['k2'] = 'some'  and array_contains(cast(v['nested1']['nested2']['a'] as array<tinyint>), 10) order by k limit 1;"""
 
@@ -117,7 +117,7 @@ parallel_pipeline_task_num=7,parallel_fragment_exec_instance_num=4,profile_level
                 properties("replication_num" = "1", "disable_auto_compaction" = "false", "enable_unique_key_merge_on_write" = "true", "variant_enable_flatten_nested" = "true");
             """
         sql """insert into var_nested2 select * from var_nested order by k limit 1024"""
-        qt_sql """select  /*+SET_VAR(batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=true,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=5,parallel_pipeline_task_num=1,profile_level=1,enable_pipeline_engine=false,enable_parallel_scan=true,parallel_scan_max_scanners_count=48,parallel_scan_min_rows_per_scanner=16384,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=12,enable_parallel_result_sink=false,enable_nereids_planner=true,rewrite_or_to_in_predicate_threshold=2,enable_function_pushdown=true,enable_common_expr_pushdown=false,enable_local_exchange=false,partition_pruning_expand_threshold=10,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_common_expr_pushdown_for_inverted_index=true,enable_delete_sub_predicate_v2=true,min_revocable_mem=33554432,fetch_remote_schema_timeout_seconds=120,max_fetch_remote_schema_tablet_count=512,enable_join_spill=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5) */  * from var_nested2 order by k limit 10;"""
+        qt_sql """select  /*+SET_VAR(batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=true,enable_distinct_streaming_aggregation=true,parallel_pipeline_task_num=1,profile_level=1,enable_pipeline_engine=false,enable_parallel_scan=true,parallel_scan_max_scanners_count=48,parallel_scan_min_rows_per_scanner=16384,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=12,enable_parallel_result_sink=false,enable_nereids_planner=true,rewrite_or_to_in_predicate_threshold=2,enable_function_pushdown=true,enable_common_expr_pushdown=false,enable_local_exchange=false,partition_pruning_expand_threshold=10,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_common_expr_pushdown_for_inverted_index=true,enable_delete_sub_predicate_v2=true,min_revocable_mem=33554432,fetch_remote_schema_timeout_seconds=120,max_fetch_remote_schema_tablet_count=512,enable_join_spill=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5) */  * from var_nested2 order by k limit 10;"""
         qt_sql """select v['nested'] from var_nested2 where k < 10 order by k limit 10;"""
         // 0. nomal explode variant array
         order_qt_explode_sql """select count(),cast(vv['xx'] as int) from var_nested lateral view explode_variant_array(v['nested']) tmp as vv where vv['xx'] = 10 group by cast(vv['xx'] as int)"""
diff --git a/regression-test/suites/variant_p0/test_sub_path_pruning.groovy b/regression-test/suites/variant_p0/test_sub_path_pruning.groovy
index c1547a0a6e039d1..daf75e4aeb4da77 100644
--- a/regression-test/suites/variant_p0/test_sub_path_pruning.groovy
+++ b/regression-test/suites/variant_p0/test_sub_path_pruning.groovy
@@ -139,7 +139,7 @@ suite("variant_sub_path_pruning", "variant_type"){
 
     // two children
     order_qt_sql """
-        select  /*+SET_VAR(batch_size=50,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=6,parallel_pipeline_task_num=2,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=false,parallel_scan_max_scanners_count=16,parallel_scan_min_rows_per_scanner=128,enable_fold_constant_by_be=false,enable_rewrite_element_at_to_slot=true,runtime_filter_type=2,enable_nereids_planner=true,rewrite_or_to_in_predicate_threshold=2,enable_function_pushdown=true,enable_common_expr_pushdown=true,enable_local_exchange=false,partition_pruning_expand_threshold=10,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_delete_sub_predicate_v2=true,min_revocable_mem=33554432,fetch_remote_schema_timeout_seconds=120,enable_join_spill=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5) */ dt['a'] as c1 from pruning_test union all select dt['a'] as c1 from pruning_test;
+        select  /*+SET_VAR(batch_size=50,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_pipeline_task_num=2,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=false,parallel_scan_max_scanners_count=16,parallel_scan_min_rows_per_scanner=128,enable_fold_constant_by_be=false,enable_rewrite_element_at_to_slot=true,runtime_filter_type=2,enable_nereids_planner=true,rewrite_or_to_in_predicate_threshold=2,enable_function_pushdown=true,enable_common_expr_pushdown=true,enable_local_exchange=false,partition_pruning_expand_threshold=10,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_delete_sub_predicate_v2=true,min_revocable_mem=33554432,fetch_remote_schema_timeout_seconds=120,enable_join_spill=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5) */ dt['a'] as c1 from pruning_test union all select dt['a'] as c1 from pruning_test;
         """
     order_qt_sql """select c1['a'] from (select dt as c1 from pruning_test union all select dt as c1 from pruning_test) v1;"""
     order_qt_sql """select c1['b'] from (select dt['a'] as c1 from pruning_test union all select dt['a'] as c1 from pruning_test) v1;"""
diff --git a/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy b/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy
index b1bbae2e3ce477f..56a0d7ddbf1f619 100644
--- a/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy
+++ b/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy
@@ -399,7 +399,7 @@ suite("test_crud_wlg") {
     sql "alter workload group test_group properties ( 'queue_timeout'='500' );"
     Thread.sleep(10000)
     test {
-        sql "select /*+SET_VAR(parallel_fragment_exec_instance_num=1)*/ * from ${table_name};"
+        sql "select /*+SET_VAR(parallel_pipeline_task_num=1)*/ * from ${table_name};"
 
         exception "query queue timeout"
     }
@@ -446,7 +446,7 @@ suite("test_crud_wlg") {
     sql "set workload_group=normal;"
     sql "alter workload group test_group properties ( 'max_concurrency'='10' );"
     Thread.sleep(10000)
-    sql "select /*+SET_VAR(parallel_fragment_exec_instance_num=1)*/ * from ${table_name};"
+    sql "select /*+SET_VAR(parallel_pipeline_task_num=1)*/ * from ${table_name};"
 
     // test workload spill property
     // 1 create group

From 56a61a95cd9fdfb13edadad5ee2c8f2788523c90 Mon Sep 17 00:00:00 2001
From: James <lijibing@selectdb.com>
Date: Thu, 9 Jan 2025 11:40:28 +0800
Subject: [PATCH 063/140] [fix](statistics)Control memory use for sample
 partition column and key column. (#46534)

### What problem does this PR solve?

When doing sample analyze for partition column and key column, BE may
encounter OOM problem. The reason is, partition column need to choose at
least one tablet in each partition to calculate the NDV and couldn't use
limit in the SQL, so when the table has large number of partitions and
each tablet in each partition is quite large, the sample SQL may try to
read too many data which will cause BE OOM.
Similarly, key column couldn't use limit as well, so when one tablet is
very large, it also could cause OOM.

This pr is try to solve this problem.
For partition columns, when the selected tablets contain more than
1000000000 (one billion) rows, we use ndv() function to read up to 5
partitions to get the NDV value of this 5 partitions, say the ndv is n.
Suppose the row count in the 5 partitions is r, and the row count of tje
table is R, the table NDV would be n * R / r.
ndv() function use hll, so it only use a small amount of memory.

For key columns, when the selected tablets contain more than 1000000000
rows, we use limit 1000000000 to control the rows to read.

Reading 1000000000 rows would use at most 8GB memory in BE, which is
acceptable.

Issue Number: close #xxx

Related PR: #xxx

Problem Summary:

### Release note

None
---
 .../doris/statistics/BaseAnalysisTask.java    |   2 +-
 .../doris/statistics/OlapAnalysisTask.java    | 315 ++++++++-----
 .../statistics/OlapAnalysisTaskTest.java      | 415 ++++++++++--------
 3 files changed, 421 insertions(+), 311 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
index e5de51b4c2d2728..6819e086ad08c68 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
@@ -81,7 +81,7 @@ public abstract class BaseAnalysisTask {
             + "         NOW() AS `update_time` "
             + " FROM `${catalogName}`.`${dbName}`.`${tblName}` ${index}";
 
-    protected static final String LINEAR_ANALYZE_TEMPLATE = " SELECT "
+    protected static final String LINEAR_ANALYZE_TEMPLATE = "SELECT "
             + "CONCAT(${tblId}, '-', ${idxId}, '-', '${colId}') AS `id`, "
             + "${catalogId} AS `catalog_id`, "
             + "${dbId} AS `db_id`, "
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
index 2dc5b774842dd1e..f0a55f9b54ee486 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
@@ -59,6 +59,12 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
             + "SUBSTRING(CAST(MAX(`${colName}`) AS STRING), 1, 1024) as max "
             + "FROM `${dbName}`.`${tblName}` ${index}";
 
+    private boolean keyColumnSampleTooManyRows = false;
+    private boolean partitionColumnSampleTooManyRows = false;
+    private boolean scanFullTable = false;
+    private static final long MAXIMUM_SAMPLE_ROWS = 1_000_000_000;
+    private static final int PARTITION_COUNT_TO_SAMPLE = 5;
+
     @VisibleForTesting
     public OlapAnalysisTask() {
     }
@@ -83,6 +89,7 @@ public void doExecute() throws Exception {
         } else {
             doFull();
         }
+        LOG.info("AnalysisTask Done {}", this.toString());
     }
 
     /**
@@ -95,77 +102,34 @@ protected void doSample() {
         if (LOG.isDebugEnabled()) {
             LOG.debug("Will do sample collection for column {}", col.getName());
         }
-        Pair<List<Long>, Long> pair = calcActualSampleTablets(tbl.isPartitionColumn(col.getName()));
-        LOG.info("Number of tablets selected {}, rows in tablets {}", pair.first.size(), pair.second);
-        List<Long> tabletIds = pair.first;
-        long totalRowCount = info.indexId == -1
-                ? tbl.getRowCount()
-                : ((OlapTable) tbl).getRowCountForIndex(info.indexId, false);
-        double scaleFactor = (double) totalRowCount / (double) pair.second;
-        // might happen if row count in fe metadata hasn't been updated yet
-        if (Double.isInfinite(scaleFactor) || Double.isNaN(scaleFactor)) {
-            LOG.debug("Scale factor is infinite or Nan, will set scale factor to 1.");
-            scaleFactor = 1;
-            tabletIds = Collections.emptyList();
-            pair.second = totalRowCount;
-        }
-        String tabletStr = tabletIds.stream()
-                .map(Object::toString)
-                .collect(Collectors.joining(", "));
         // Get basic stats, including min and max.
-        ResultRow basicStats = collectBasicStat();
-        String min = StatisticsUtil.escapeSQL(basicStats != null && basicStats.getValues().size() > 0
-                ? basicStats.get(0) : null);
-        String max = StatisticsUtil.escapeSQL(basicStats != null && basicStats.getValues().size() > 1
-                ? basicStats.get(1) : null);
-
-        boolean limitFlag = false;
-        long rowsToSample = pair.second;
+        ResultRow minMax = collectMinMax();
+        String min = StatisticsUtil.escapeSQL(minMax != null && minMax.getValues().size() > 0
+                ? minMax.get(0) : null);
+        String max = StatisticsUtil.escapeSQL(minMax != null && minMax.getValues().size() > 1
+                ? minMax.get(1) : null);
+
         Map<String, String> params = buildSqlParams();
-        params.put("scaleFactor", String.valueOf(scaleFactor));
-        params.put("sampleHints", tabletStr.isEmpty() ? "" : String.format("TABLET(%s)", tabletStr));
-        params.put("ndvFunction", getNdvFunction(String.valueOf(totalRowCount)));
         params.put("min", StatisticsUtil.quote(min));
         params.put("max", StatisticsUtil.quote(max));
-        params.put("rowCount", String.valueOf(totalRowCount));
-        params.put("type", col.getType().toString());
-        params.put("limit", "");
-        if (needLimit()) {
-            // If the tablets to be sampled are too large, use limit to control the rows to read, and re-calculate
-            // the scaleFactor.
-            rowsToSample = Math.min(getSampleRows(), pair.second);
-            // Empty table doesn't need to limit.
-            if (rowsToSample > 0) {
-                limitFlag = true;
-                params.put("limit", "limit " + rowsToSample);
-                params.put("scaleFactor", String.valueOf(scaleFactor * (double) pair.second / rowsToSample));
-            }
-        }
+        long tableRowCount = info.indexId == -1
+                ? tbl.getRowCount()
+                : ((OlapTable) tbl).getRowCountForIndex(info.indexId, false);
+        getSampleParams(params, tableRowCount);
         StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
         String sql;
         if (useLinearAnalyzeTemplate()) {
-            // For single unique key, use count as ndv.
-            if (isSingleUniqueKey()) {
-                params.put("ndvFunction", String.valueOf(totalRowCount));
-            } else {
-                params.put("ndvFunction", "ROUND(NDV(`${colName}`) * ${scaleFactor})");
-            }
             sql = stringSubstitutor.replace(LINEAR_ANALYZE_TEMPLATE);
         } else {
-            params.put("dataSizeFunction", getDataSizeFunction(col, true));
-            params.put("subStringColName", getStringTypeColName(col));
             sql = stringSubstitutor.replace(DUJ1_ANALYZE_TEMPLATE);
         }
-        LOG.info("Sample for column [{}]. Total rows [{}], rows to sample [{}], scale factor [{}], "
-                + "limited [{}], distribute column [{}], partition column [{}], key column [{}], "
-                + "is single unique key [{}]",
-                col.getName(), params.get("rowCount"), rowsToSample, params.get("scaleFactor"),
-                limitFlag, tbl.isDistributionColumn(col.getName()),
-                tbl.isPartitionColumn(col.getName()), col.isKey(), isSingleUniqueKey());
+        LOG.info("Analyze param: scanFullTable {}, partitionColumnTooMany {}, keyColumnTooMany {}",
+                scanFullTable, partitionColumnSampleTooManyRows, keyColumnSampleTooManyRows);
+        LOG.debug(sql);
         runQuery(sql);
     }
 
-    protected ResultRow collectBasicStat() {
+    protected ResultRow collectMinMax() {
         // Agg table value columns has no zone map.
         // For these columns, skip collecting min and max value to avoid scan whole table.
         if (((OlapTable) tbl).getKeysType().equals(KeysType.AGG_KEYS) && !col.isKey()) {
@@ -187,10 +151,140 @@ protected ResultRow collectBasicStat() {
             }
             // Release the reference to stmtExecutor, reduce memory usage.
             stmtExecutor = null;
+        } catch (Exception e) {
+            LOG.info("Failed to collect basic stat {}. Reason {}", sql, e.getMessage());
+            throw e;
         }
         return resultRow;
     }
 
+    /**
+     * Select the tablets to read.
+     * @return Pair of tablet id list and how many rows are going to read.
+     */
+    protected Pair<List<Long>, Long> getSampleTablets() {
+        long targetSampleRows = getSampleRows();
+        OlapTable olapTable = (OlapTable) tbl;
+        boolean forPartitionColumn = tbl.isPartitionColumn(col.getName());
+        long avgTargetRowsPerPartition = targetSampleRows / Math.max(olapTable.getPartitions().size(), 1);
+        List<Long> sampleTabletIds = new ArrayList<>();
+        long selectedRows = 0;
+        boolean enough = false;
+        // Sort the partitions to get stable result.
+        List<Partition> sortedPartitions = olapTable.getPartitions().stream().sorted(
+                Comparator.comparing(Partition::getName)).collect(Collectors.toList());
+        for (Partition p : sortedPartitions) {
+            MaterializedIndex materializedIndex = info.indexId == -1 ? p.getBaseIndex() : p.getIndex(info.indexId);
+            if (materializedIndex == null) {
+                continue;
+            }
+            List<Long> ids = materializedIndex.getTabletIdsInOrder();
+            if (ids.isEmpty()) {
+                continue;
+            }
+            long avgRowsPerTablet = Math.max(materializedIndex.getRowCount() / ids.size(), 1);
+            long tabletCounts = Math.max(avgTargetRowsPerPartition / avgRowsPerTablet
+                    + (avgTargetRowsPerPartition % avgRowsPerTablet != 0 ? 1 : 0), 1);
+            tabletCounts = Math.min(tabletCounts, ids.size());
+            long seek = tableSample.getSeek() != -1 ? tableSample.getSeek()
+                    : (long) (new SecureRandom().nextDouble() * ids.size());
+            for (int i = 0; i < tabletCounts; i++) {
+                int seekTid = (int) ((i + seek) % ids.size());
+                long tabletId = ids.get(seekTid);
+                sampleTabletIds.add(tabletId);
+                long tabletRows = materializedIndex.getTablet(tabletId).getMinReplicaRowCount(p.getVisibleVersion());
+                if (tabletRows > 0) {
+                    selectedRows += tabletRows;
+                    // For regular column, will stop adding more tablets when selected tablets'
+                    // row count is more than the target sample rows.
+                    // But for partition columns, will not stop adding. For ndv sample accuracy,
+                    // better to choose at least one tablet in each partition.
+                    if (selectedRows >= targetSampleRows && !forPartitionColumn) {
+                        enough = true;
+                        break;
+                    }
+                }
+            }
+            if (enough) {
+                break;
+            }
+        }
+        if (selectedRows < targetSampleRows) {
+            scanFullTable = true;
+        } else if (forPartitionColumn && selectedRows > MAXIMUM_SAMPLE_ROWS) {
+            // If the selected tablets for partition column contain too many rows, change to linear sample.
+            partitionColumnSampleTooManyRows = true;
+            sampleTabletIds.clear();
+            Collections.shuffle(sortedPartitions);
+            selectedRows = pickSamplePartition(sortedPartitions, sampleTabletIds);
+        } else if (col.isKey() && selectedRows > MAXIMUM_SAMPLE_ROWS) {
+            // For key column, if a single tablet contains too many rows, need to use limit to control rows to read.
+            // In most cases, a single tablet shouldn't contain more than MAXIMUM_SAMPLE_ROWS, in this case, we
+            // don't use limit for key column for ndv accuracy reason.
+            keyColumnSampleTooManyRows = true;
+        }
+        return Pair.of(sampleTabletIds, selectedRows);
+    }
+
+    /**
+     * Get the sql params for this sample task.
+     * @param params Sql params to use in analyze task.
+     * @param tableRowCount BE reported table/index row count.
+     */
+    protected void getSampleParams(Map<String, String> params, long tableRowCount) {
+        long targetSampleRows = getSampleRows();
+        params.put("rowCount", String.valueOf(tableRowCount));
+        params.put("type", col.getType().toString());
+        params.put("limit", "");
+
+        // If table row count is less than the target sample row count, simple scan the full table.
+        if (tableRowCount <= targetSampleRows) {
+            params.put("scaleFactor", "1");
+            params.put("sampleHints", "");
+            params.put("ndvFunction", "ROUND(NDV(`${colName}`) * ${scaleFactor})");
+            scanFullTable = true;
+            return;
+        }
+        Pair<List<Long>, Long> sampleTabletsInfo = getSampleTablets();
+        String tabletStr = sampleTabletsInfo.first.stream()
+                .map(Object::toString)
+                .collect(Collectors.joining(", "));
+        String sampleHints = scanFullTable ? "" : String.format("TABLET(%s)", tabletStr);
+        params.put("sampleHints", sampleHints);
+        long selectedRows = sampleTabletsInfo.second;
+        long finalScanRows = selectedRows;
+        double scaleFactor = scanFullTable ? 1 : (double) tableRowCount / finalScanRows;
+        params.put("scaleFactor", String.valueOf(scaleFactor));
+
+        // If the tablets to be sampled are too large, use limit to control the rows to read, and re-calculate
+        // the scaleFactor.
+        if (needLimit()) {
+            finalScanRows = Math.min(targetSampleRows, selectedRows);
+            if (col.isKey() && keyColumnSampleTooManyRows) {
+                finalScanRows = MAXIMUM_SAMPLE_ROWS;
+            }
+            // Empty table doesn't need to limit.
+            if (finalScanRows > 0) {
+                scaleFactor = (double) tableRowCount / finalScanRows;
+                params.put("limit", "limit " + finalScanRows);
+                params.put("scaleFactor", String.valueOf(scaleFactor));
+            }
+        }
+        // Set algorithm related params.
+        if (useLinearAnalyzeTemplate()) {
+            // For single unique key, use count as ndv.
+            if (isSingleUniqueKey()) {
+                params.put("ndvFunction", String.valueOf(tableRowCount));
+            } else {
+                params.put("ndvFunction", "ROUND(NDV(`${colName}`) * ${scaleFactor})");
+            }
+        } else {
+            params.put("ndvFunction", getNdvFunction(String.valueOf(tableRowCount)));
+            params.put("dataSizeFunction", getDataSizeFunction(col, true));
+            params.put("subStringColName", getStringTypeColName(col));
+        }
+    }
+
     protected void doFull() throws Exception {
         if (LOG.isDebugEnabled()) {
             LOG.debug("Will do full collection for column {}", col.getName());
@@ -279,72 +373,29 @@ protected String getIndex() {
         }
     }
 
-    // Get sample tablets id and sample row count
-    protected Pair<List<Long>, Long> calcActualSampleTablets(boolean forPartitionColumn) {
-        // Below code copied from OlapScanNode.java
-        long sampleRows; // The total number of sample rows
-        long totalRows = 0; // The total number of partition rows hit
-        long totalTablet = 0; // The total number of tablets in the hit partition
-        OlapTable olapTable = (OlapTable) tbl;
-        sampleRows = getSampleRows();
-
-        // calculate the number of tablets by each partition
-        long avgRowsPerPartition = sampleRows / Math.max(olapTable.getPartitions().size(), 1);
-        List<Long> sampleTabletIds = new ArrayList<>();
-        long actualSampledRowCount = 0;
-        boolean enough = false;
-        List<Partition> sortedPartitions = olapTable.getPartitions().stream().sorted(
-                Comparator.comparing(Partition::getName)).collect(Collectors.toList());
-        for (Partition p : sortedPartitions) {
-            MaterializedIndex materializedIndex = info.indexId == -1 ? p.getBaseIndex() : p.getIndex(info.indexId);
-            if (materializedIndex == null) {
-                continue;
-            }
-            List<Long> ids = materializedIndex.getTabletIdsInOrder();
-            if (ids.isEmpty()) {
-                continue;
+    protected long pickSamplePartition(List<Partition> partitions, List<Long> pickedTabletIds) {
+        long averageRowsPerPartition = tbl.getRowCount() / partitions.size();
+        long indexId = info.indexId == -1 ? ((OlapTable) tbl).getBaseIndexId() : info.indexId;
+        long pickedRows = 0;
+        int pickedPartitionCount = 0;
+        for (Partition p : partitions) {
+            long partitionRowCount = p.getRowCount();
+            if (partitionRowCount >= averageRowsPerPartition) {
+                pickedRows += partitionRowCount;
+                pickedPartitionCount++;
+                MaterializedIndex materializedIndex = p.getIndex(indexId);
+                pickedTabletIds.addAll(materializedIndex.getTabletIdsInOrder());
             }
-
-            // Skip partitions with row count < row count / 2 expected to be sampled per partition.
-            // It can be expected to sample a smaller number of partitions to avoid uneven distribution
-            // of sampling results.
-            if (materializedIndex.getRowCount() < (avgRowsPerPartition / 2) && !forPartitionColumn) {
-                continue;
-            }
-            long avgRowsPerTablet = Math.max(materializedIndex.getRowCount() / ids.size(), 1);
-            long tabletCounts = Math.max(
-                    avgRowsPerPartition / avgRowsPerTablet + (avgRowsPerPartition % avgRowsPerTablet != 0 ? 1 : 0), 1);
-            tabletCounts = Math.min(tabletCounts, ids.size());
-            long seek = tableSample.getSeek() != -1
-                    ? tableSample.getSeek() : (long) (new SecureRandom().nextDouble() * ids.size());
-            for (int i = 0; i < tabletCounts; i++) {
-                int seekTid = (int) ((i + seek) % ids.size());
-                long tabletId = ids.get(seekTid);
-                sampleTabletIds.add(tabletId);
-                actualSampledRowCount += materializedIndex.getTablet(tabletId)
-                        .getMinReplicaRowCount(p.getVisibleVersion());
-                if (actualSampledRowCount >= sampleRows && !forPartitionColumn) {
-                    enough = true;
-                    break;
-                }
-            }
-            totalRows += materializedIndex.getRowCount();
-            totalTablet += ids.size();
-            if (enough) {
+            if (pickedRows >= MAXIMUM_SAMPLE_ROWS || pickedPartitionCount > PARTITION_COUNT_TO_SAMPLE) {
                 break;
             }
         }
+        return pickedRows;
+    }
 
-        // all hit, direct full
-        if (totalRows < sampleRows) {
-            // can't fill full sample rows
-            sampleTabletIds.clear();
-            actualSampledRowCount = 0;
-        } else if (sampleTabletIds.size() == totalTablet && !enough) {
-            sampleTabletIds.clear();
-            actualSampledRowCount = 0;
-        }
-        return Pair.of(sampleTabletIds, actualSampledRowCount);
+    @VisibleForTesting
+    protected void setTable(OlapTable table) {
+        tbl = table;
     }
 
     /**
@@ -352,8 +403,11 @@ protected Pair<List<Long>, Long> calcActualSampleTablets(boolean forPartitionCol
      * @return Return true when need to limit.
      */
     protected boolean needLimit() {
+        if (scanFullTable) {
+            return false;
+        }
         // Key column is sorted, use limit will cause the ndv not accurate enough, so skip key columns.
-        if (col.isKey()) {
+        if (col.isKey() && !keyColumnSampleTooManyRows) {
             return false;
         }
         // Partition column need to scan tablets from all partitions.
@@ -379,6 +433,9 @@ protected long getSampleRows() {
      * @return True for single unique key column and single distribution column.
      */
     protected boolean useLinearAnalyzeTemplate() {
+        if (partitionColumnSampleTooManyRows || scanFullTable) {
+            return true;
+        }
         if (isSingleUniqueKey()) {
             return true;
         }
@@ -422,4 +479,24 @@ protected boolean isSingleUniqueKey() {
     protected String concatColumnStatsId() {
         return info.tblId + "-" + info.indexId + "-" + info.colName;
     }
+
+    @VisibleForTesting
+    public void setKeyColumnSampleTooManyRows(boolean value) {
+        keyColumnSampleTooManyRows = value;
+    }
+
+    @VisibleForTesting
+    public void setPartitionColumnSampleTooManyRows(boolean value) {
+        partitionColumnSampleTooManyRows = value;
+    }
+
+    @VisibleForTesting
+    public void setScanFullTable(boolean value) {
+        scanFullTable = value;
+    }
+
+    @VisibleForTesting
+    public boolean scanFullTable() {
+        return scanFullTable;
+    }
 }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java
index ce47143dbb1548a..9bda917b34ea1a0 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java
@@ -20,21 +20,23 @@
 import org.apache.doris.analysis.TableSample;
 import org.apache.doris.catalog.Column;
 import org.apache.doris.catalog.DatabaseIf;
+import org.apache.doris.catalog.MaterializedIndex;
 import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.Partition;
 import org.apache.doris.catalog.PartitionInfo;
 import org.apache.doris.catalog.PartitionType;
 import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.catalog.RandomDistributionInfo;
 import org.apache.doris.catalog.TableIf;
 import org.apache.doris.catalog.Type;
 import org.apache.doris.common.Pair;
 import org.apache.doris.datasource.CatalogIf;
-import org.apache.doris.qe.AutoCloseConnectContext;
 import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod;
 import org.apache.doris.statistics.AnalysisInfo.JobType;
 import org.apache.doris.statistics.util.StatisticsUtil;
 
 import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
+import com.google.common.collect.Maps;
 import mockit.Expectations;
 import mockit.Mock;
 import mockit.MockUp;
@@ -43,9 +45,8 @@
 import org.junit.jupiter.api.Test;
 
 import java.util.ArrayList;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
+import java.util.Map;
 
 public class OlapAnalysisTaskTest {
 
@@ -93,13 +94,12 @@ public long getDataSize(boolean singleReplica) {
     }
 
     @Test
-    public void testManualSampleNonDistributeKey(@Mocked CatalogIf catalogIf, @Mocked DatabaseIf databaseIf, @Mocked OlapTable tableIf)
-            throws Exception {
+    public void testKeyColumnUseLimitAndNot(@Mocked CatalogIf catalogIf, @Mocked DatabaseIf databaseIf, @Mocked OlapTable tableIf) {
 
         new Expectations() {
             {
                 tableIf.getRowCount();
-                result = 500;
+                result = 20000000;
                 tableIf.getId();
                 result = 30001;
                 catalogIf.getId();
@@ -113,12 +113,7 @@ public void testManualSampleNonDistributeKey(@Mocked CatalogIf catalogIf, @Mocke
 
         new MockUp<OlapAnalysisTask>() {
             @Mock
-            public Pair<List<Long>, Long> calcActualSampleTablets() {
-                return Pair.of(Lists.newArrayList(), 100L);
-            }
-
-            @Mock
-            public ResultRow collectBasicStat() {
+            public ResultRow collectMinMax() {
                 List<String> values = Lists.newArrayList();
                 values.add("1");
                 values.add("2");
@@ -126,42 +121,29 @@ public ResultRow collectBasicStat() {
             }
 
             @Mock
-            public void runQuery(String sql) {
-                Assertions.assertEquals("SELECT CONCAT('30001', '-', '-1', '-', 'null') "
-                        + "AS `id`, 10001 AS `catalog_id`, 20001 AS `db_id`, 30001 AS `tbl_id`, "
-                        + "-1 AS `idx_id`, 'null' AS `col_id`, NULL AS `part_id`, 500 AS"
-                        + " `row_count`, SUM(`t1`.`count`) * COUNT(1) / (SUM(`t1`.`count`)"
-                        + " - SUM(IF(`t1`.`count` = 1, 1, 0)) + SUM(IF(`t1`.`count` = 1, 1, 0))"
-                        + " * SUM(`t1`.`count`) / 500) as `ndv`, IFNULL(SUM(IF(`t1`.`column_key`"
-                        + " IS NULL, `t1`.`count`, 0)), 0) * 5.0 as `null_count`, "
-                        + "SUBSTRING(CAST('1' AS STRING), 1, 1024) AS `min`,"
-                        + " SUBSTRING(CAST('2' AS STRING), 1, 1024) AS `max`, "
-                        + "SUM(t1.count) * 4 * 5.0 AS `data_size`, NOW() "
-                        + "FROM (     SELECT t0.`colValue` as `column_key`, COUNT(1) "
-                        + "as `count`, SUM(`len`) as `column_length`     FROM         "
-                        + "(SELECT `null` AS `colValue`, LENGTH(`null`) as `len`         "
-                        + "FROM `catalogName`.`${dbName}`.`null`"
-                        + "   limit 100) as `t0`     GROUP BY `t0`.`colValue` ) as `t1` ", sql);
-                return;
-            }
-        };
+            protected void getSampleParams(Map<String, String> params, long tableRowCount) {}
 
-        new MockUp<StatisticsUtil>() {
             @Mock
-            public AutoCloseConnectContext buildConnectContext(boolean scanLimit) {
-                return null;
+            protected boolean useLinearAnalyzeTemplate() {
+                return true;
             }
-        };
 
-        new MockUp<OlapTable>() {
-            @Mock
-            public Set<String> getDistributionColumnNames() {
-                return Sets.newHashSet();
+                @Mock
+            public void runQuery(String sql) {
+                Assertions.assertEquals("SELECT CONCAT(30001, '-', -1, '-', 'null') AS `id`, "
+                        + "10001 AS `catalog_id`, 20001 AS `db_id`, 30001 AS `tbl_id`, -1 AS `idx_id`, "
+                        + "'null' AS `col_id`, NULL AS `part_id`, ${rowCount} AS `row_count`, "
+                        + "${ndvFunction} as `ndv`, ROUND(SUM(CASE WHEN `null` IS NULL THEN 1 ELSE 0 END) * ${scaleFactor}) AS `null_count`, "
+                        + "SUBSTRING(CAST('1' AS STRING), 1, 1024) AS `min`, SUBSTRING(CAST('2' AS STRING), 1, 1024) AS `max`, "
+                        + "COUNT(1) * 4 * ${scaleFactor} AS `data_size`, NOW() FROM "
+                        + "( SELECT * FROM `catalogName`.`${dbName}`.`null`  ${sampleHints} ${limit})  as t", sql);
+                return;
             }
         };
 
         OlapAnalysisTask olapAnalysisTask = new OlapAnalysisTask();
-        olapAnalysisTask.col = new Column("test", PrimitiveType.INT);
+        olapAnalysisTask.col = new Column("test", Type.fromPrimitiveType(PrimitiveType.INT),
+            true, null, null, null);
         olapAnalysisTask.tbl = tableIf;
         AnalysisInfoBuilder analysisInfoBuilder = new AnalysisInfoBuilder();
         analysisInfoBuilder.setJobType(AnalysisInfo.JobType.MANUAL);
@@ -170,220 +152,271 @@ public Set<String> getDistributionColumnNames() {
         olapAnalysisTask.db = databaseIf;
         olapAnalysisTask.tableSample = new TableSample(false, 100L);
         olapAnalysisTask.doSample();
-    }
-
-    @Test
-    public void testManualSampleDistributeKey(@Mocked CatalogIf catalogIf, @Mocked DatabaseIf databaseIf, @Mocked OlapTable tableIf)
-            throws Exception {
-
-        new Expectations() {
-            {
-                tableIf.getRowCount();
-                result = 500;
-                tableIf.getId();
-                result = 30001;
-                catalogIf.getId();
-                result = 10001;
-                catalogIf.getName();
-                result = "catalogName";
-                databaseIf.getId();
-                result = 20001;
-            }
-        };
 
         new MockUp<OlapAnalysisTask>() {
-            @Mock
-            public Pair<List<Long>, Long> calcActualSampleTablets() {
-                return Pair.of(Lists.newArrayList(), 100L);
-            }
-
-            @Mock
-            public ResultRow collectBasicStat() {
-                List<String> values = Lists.newArrayList();
-                values.add("1");
-                values.add("2");
-                return new ResultRow(values);
-            }
-
             @Mock
             public void runQuery(String sql) {
-                Assertions.assertEquals(" SELECT CONCAT(30001, '-', -1, '-', 'null') AS `id`, "
-                        + "10001 AS `catalog_id`, 20001 AS `db_id`, 30001 AS `tbl_id`, "
-                        + "-1 AS `idx_id`, 'null' AS `col_id`, NULL AS `part_id`, "
-                        + "500 AS `row_count`, ROUND(NDV(`null`) * 5.0) as `ndv`, "
-                        + "ROUND(SUM(CASE WHEN `null` IS NULL THEN 1 ELSE 0 END) * 5.0) "
-                        + "AS `null_count`, SUBSTRING(CAST('1' AS STRING), 1, 1024) AS `min`, "
-                        + "SUBSTRING(CAST('2' AS STRING), 1, 1024) AS `max`, "
-                        + "SUM(LENGTH(`null`)) * 5.0 AS `data_size`, NOW() "
-                        + "FROM ( SELECT * FROM `catalogName`.`${dbName}`.`null`   limit 100)  as t", sql);
+                Assertions.assertEquals("SELECT CONCAT('30001', '-', '-1', '-', 'null') AS `id`, "
+                        + "10001 AS `catalog_id`, 20001 AS `db_id`, 30001 AS `tbl_id`, -1 AS `idx_id`, "
+                        + "'null' AS `col_id`, NULL AS `part_id`, ${rowCount} AS `row_count`, ${ndvFunction} as `ndv`, "
+                        + "IFNULL(SUM(IF(`t1`.`column_key` IS NULL, `t1`.`count`, 0)), 0) * ${scaleFactor} as `null_count`, "
+                        + "SUBSTRING(CAST('1' AS STRING), 1, 1024) AS `min`, SUBSTRING(CAST('2' AS STRING), 1, 1024) AS `max`, "
+                        + "COUNT(1) * 4 * ${scaleFactor} AS `data_size`, NOW() "
+                        + "FROM (     SELECT t0.`colValue` as `column_key`, COUNT(1) as `count`, SUM(`len`) as `column_length`     "
+                        + "FROM         (SELECT ${subStringColName} AS `colValue`, LENGTH(`null`) as `len`         "
+                        + "FROM `catalogName`.`${dbName}`.`null`  ${sampleHints} ${limit}) as `t0`     GROUP BY `t0`.`colValue` ) as `t1` ", sql);
                 return;
             }
-        };
 
-        new MockUp<StatisticsUtil>() {
             @Mock
-            public AutoCloseConnectContext buildConnectContext(boolean scanLimit) {
-                return null;
+            protected boolean useLinearAnalyzeTemplate() {
+                return false;
             }
         };
+        olapAnalysisTask.doSample();
+    }
+
+    @Test
+    public void testNeedLimitFalse(@Mocked CatalogIf catalogIf, @Mocked DatabaseIf databaseIf, @Mocked OlapTable tableIf)
+            throws Exception {
 
         new MockUp<OlapTable>() {
             @Mock
-            public Set<String> getDistributionColumnNames() {
-                HashSet<String> cols = Sets.newHashSet();
-                cols.add("test");
-                return cols;
+            public PartitionInfo getPartitionInfo() {
+                ArrayList<Column> columns = Lists.newArrayList();
+                columns.add(new Column("test", PrimitiveType.STRING));
+                return new PartitionInfo(PartitionType.RANGE, columns);
             }
 
             @Mock
-            public boolean isDistributionColumn(String columnName) {
+            public boolean isPartitionColumn(String columnName) {
                 return true;
             }
         };
 
         OlapAnalysisTask olapAnalysisTask = new OlapAnalysisTask();
-        olapAnalysisTask.col = new Column("test", PrimitiveType.STRING);
+        olapAnalysisTask.col = new Column("test", Type.fromPrimitiveType(PrimitiveType.STRING),
+            true, null, null, null);
         olapAnalysisTask.tbl = tableIf;
-        AnalysisInfoBuilder analysisInfoBuilder = new AnalysisInfoBuilder();
-        analysisInfoBuilder.setJobType(AnalysisInfo.JobType.MANUAL);
-        olapAnalysisTask.info = analysisInfoBuilder.build();
-        olapAnalysisTask.catalog = catalogIf;
-        olapAnalysisTask.db = databaseIf;
-        olapAnalysisTask.tableSample = new TableSample(false, 100L);
-        olapAnalysisTask.doSample();
+        Assertions.assertFalse(olapAnalysisTask.needLimit());
+
+        olapAnalysisTask.col = new Column("test", Type.fromPrimitiveType(PrimitiveType.STRING),
+            false, null, null, null);
+        Assertions.assertFalse(olapAnalysisTask.needLimit());
     }
 
     @Test
-    public void testManualSampleTwoDistributeKey(@Mocked CatalogIf catalogIf, @Mocked DatabaseIf databaseIf, @Mocked OlapTable tableIf)
+    public void testNeedLimitTrue(@Mocked CatalogIf catalogIf, @Mocked DatabaseIf databaseIf, @Mocked OlapTable tableIf)
             throws Exception {
 
-        new Expectations() {
-            {
-                tableIf.getRowCount();
-                result = 500;
-                tableIf.getId();
-                result = 30001;
-                catalogIf.getId();
-                result = 10001;
-                catalogIf.getName();
-                result = "catalogName";
-                databaseIf.getId();
-                result = 20001;
+        new MockUp<OlapTable>() {
+            @Mock
+            public PartitionInfo getPartitionInfo() {
+                ArrayList<Column> columns = Lists.newArrayList();
+                columns.add(new Column("NOFOUND", PrimitiveType.STRING));
+                return new PartitionInfo(PartitionType.RANGE, columns);
             }
         };
 
-        new MockUp<OlapAnalysisTask>() {
+        OlapAnalysisTask olapAnalysisTask = new OlapAnalysisTask();
+        olapAnalysisTask.tbl = tableIf;
+        olapAnalysisTask.col = new Column("test", Type.fromPrimitiveType(PrimitiveType.STRING),
+            false, null, null, null);
+        Assertions.assertTrue(olapAnalysisTask.needLimit());
+
+        olapAnalysisTask.col = new Column("test", Type.fromPrimitiveType(PrimitiveType.STRING),
+            true, null, null, null);
+        olapAnalysisTask.setKeyColumnSampleTooManyRows(true);
+        Assertions.assertTrue(olapAnalysisTask.needLimit());
+    }
+
+    @Test
+    public void testPickSamplePartition() {
+        OlapAnalysisTask task = new OlapAnalysisTask();
+        AnalysisInfoBuilder builder = new AnalysisInfoBuilder();
+        task.info = builder.setIndexId(-1L).build();
+        task.setTable(new OlapTable());
+        Partition p1 = new Partition(1, "p1", new MaterializedIndex(), new RandomDistributionInfo());
+        Partition p2 = new Partition(2, "p2", new MaterializedIndex(), new RandomDistributionInfo());
+        Partition p3 = new Partition(3, "p3", new MaterializedIndex(), new RandomDistributionInfo());
+        List<Partition> partitions = Lists.newArrayList();
+        partitions.add(p1);
+        partitions.add(p2);
+        partitions.add(p3);
+        List<Long> ids = Lists.newArrayList();
+
+        new MockUp<OlapTable>() {
             @Mock
-            public Pair<List<Long>, Long> calcActualSampleTablets() {
-                return Pair.of(Lists.newArrayList(), 100L);
+            public long getRowCount() {
+                return 1000000000L;
             }
+        };
 
+        long[] partitionRows = new long[3];
+        partitionRows[0] = 400000000L;
+        partitionRows[1] = 100000000L;
+        partitionRows[2] = 500000000L;
+        final int[] i = {0};
+        new MockUp<Partition>() {
             @Mock
-            public ResultRow collectBasicStat() {
-                List<String> values = Lists.newArrayList();
-                values.add("1");
-                values.add("2");
-                return new ResultRow(values);
+            public long getRowCount() {
+                return partitionRows[i[0]++];
             }
 
             @Mock
-            public void runQuery(String sql) {
-                System.out.println(sql);
-                Assertions.assertEquals("SELECT CONCAT('30001', '-', '-1', '-', 'null') "
-                        + "AS `id`, 10001 AS `catalog_id`, 20001 AS `db_id`, 30001 AS `tbl_id`, "
-                        + "-1 AS `idx_id`, 'null' AS `col_id`, NULL AS `part_id`,"
-                        + " 500 AS `row_count`, SUM(`t1`.`count`) * COUNT(1) / (SUM(`t1`.`count`) "
-                        + "- SUM(IF(`t1`.`count` = 1, 1, 0)) + SUM(IF(`t1`.`count` = 1, 1, 0)) * "
-                        + "SUM(`t1`.`count`) / 500) as `ndv`, IFNULL(SUM(IF(`t1`.`column_key` "
-                        + "IS NULL, `t1`.`count`, 0)), 0) * 5.0 as `null_count`, "
-                        + "SUBSTRING(CAST('1' AS STRING), 1, 1024) AS `min`, "
-                        + "SUBSTRING(CAST('2' AS STRING), 1, 1024) AS `max`, "
-                        + "SUM(`column_length`) * 5.0 AS `data_size`, NOW() "
-                        + "FROM (     SELECT t0.`colValue` as `column_key`, COUNT(1) as `count`, SUM(`len`) as "
-                        + "`column_length`     FROM         (SELECT xxhash_64(SUBSTRING(CAST(`null` AS STRING), 1, 1024)) "
-                        + "AS `colValue`, LENGTH(`null`) as `len`"
-                        + "         FROM `catalogName`.`${dbName}`.`null`   limit 100) as `t0`     "
-                        + "GROUP BY `t0`.`colValue` ) as `t1` ", sql);
-                return;
+            public MaterializedIndex getIndex(long indexId) {
+                return new MaterializedIndex();
             }
         };
 
-        new MockUp<StatisticsUtil>() {
+        final int[] j = {0};
+        new MockUp<MaterializedIndex>() {
             @Mock
-            public AutoCloseConnectContext buildConnectContext(boolean scanLimit) {
-                return null;
+            public List<Long> getTabletIdsInOrder() {
+                List<Long> ret = new ArrayList<>();
+                ret.add((long) j[0]++);
+                ret.add((long) j[0]++);
+                return ret;
             }
         };
+        long rows = task.pickSamplePartition(partitions, ids);
+        Assertions.assertEquals(900000000, rows);
+        Assertions.assertEquals(4, ids.size());
+        Assertions.assertEquals(0, ids.get(0));
+        Assertions.assertEquals(1, ids.get(1));
+        Assertions.assertEquals(2, ids.get(2));
+        Assertions.assertEquals(3, ids.get(3));
+    }
 
-        new MockUp<OlapTable>() {
+    @Test
+    public void testUseLinearAnalyzeTemplate() {
+        OlapAnalysisTask task = new OlapAnalysisTask();
+        task.setPartitionColumnSampleTooManyRows(true);
+        Assertions.assertTrue(task.useLinearAnalyzeTemplate());
+
+        task.setPartitionColumnSampleTooManyRows(false);
+        task.setScanFullTable(true);
+        Assertions.assertTrue(task.useLinearAnalyzeTemplate());
+
+        task.setScanFullTable(false);
+        task.setPartitionColumnSampleTooManyRows(false);
+        new MockUp<OlapAnalysisTask>() {
             @Mock
-            public Set<String> getDistributionColumnNames() {
-                HashSet<String> cols = Sets.newHashSet();
-                cols.add("test1");
-                cols.add("test2");
-                return cols;
+            protected boolean isSingleUniqueKey() {
+                return true;
             }
         };
-
-        OlapAnalysisTask olapAnalysisTask = new OlapAnalysisTask();
-        olapAnalysisTask.col = new Column("test1", PrimitiveType.STRING);
-        olapAnalysisTask.tbl = tableIf;
-        AnalysisInfoBuilder analysisInfoBuilder = new AnalysisInfoBuilder();
-        analysisInfoBuilder.setJobType(AnalysisInfo.JobType.MANUAL);
-        olapAnalysisTask.info = analysisInfoBuilder.build();
-        olapAnalysisTask.catalog = catalogIf;
-        olapAnalysisTask.db = databaseIf;
-        olapAnalysisTask.tableSample = new TableSample(false, 100L);
-        olapAnalysisTask.doSample();
+        Assertions.assertTrue(task.useLinearAnalyzeTemplate());
     }
 
     @Test
-    public void testNeedLimitFalse(@Mocked CatalogIf catalogIf, @Mocked DatabaseIf databaseIf, @Mocked OlapTable tableIf)
-            throws Exception {
+    public void testGetSampleParams() {
+        OlapAnalysisTask task = new OlapAnalysisTask();
+        Map<String, String> params = Maps.newHashMap();
+        new MockUp<OlapAnalysisTask>() {
+            @Mock
+            protected long getSampleRows() {
+                return 100;
+            }
 
-        new MockUp<OlapTable>() {
             @Mock
-            public PartitionInfo getPartitionInfo() {
-                ArrayList<Column> columns = Lists.newArrayList();
-                columns.add(new Column("test", PrimitiveType.STRING));
-                return new PartitionInfo(PartitionType.RANGE, columns);
+            protected Pair<List<Long>, Long> getSampleTablets() {
+                List<Long> ids = Lists.newArrayList();
+                ids.add(1L);
+                ids.add(2L);
+                return Pair.of(ids, 100L);
             }
 
             @Mock
-            public boolean isPartitionColumn(String columnName) {
-                return true;
+            protected boolean needLimit() {
+                return false;
+            }
+
+            @Mock
+            protected boolean useLinearAnalyzeTemplate() {
+                return false;
             }
         };
+        task.col = new Column("test", PrimitiveType.INT);
+        task.getSampleParams(params, 10);
+        Assertions.assertTrue(task.scanFullTable());
+        Assertions.assertEquals("1", params.get("scaleFactor"));
+        Assertions.assertEquals("", params.get("sampleHints"));
+        Assertions.assertEquals("ROUND(NDV(`${colName}`) * ${scaleFactor})", params.get("ndvFunction"));
+        params.clear();
+
+        task = new OlapAnalysisTask();
+        task.col = new Column("test", PrimitiveType.INT);
+        task.getSampleParams(params, 1000);
+        Assertions.assertEquals("10.0", params.get("scaleFactor"));
+        Assertions.assertEquals("TABLET(1, 2)", params.get("sampleHints"));
+        Assertions.assertEquals("SUM(`t1`.`count`) * COUNT(1) / (SUM(`t1`.`count`) - SUM(IF(`t1`.`count` = 1, 1, 0)) + SUM(IF(`t1`.`count` = 1, 1, 0)) * SUM(`t1`.`count`) / 1000)", params.get("ndvFunction"));
+        Assertions.assertEquals("SUM(t1.count) * 4", params.get("dataSizeFunction"));
+        Assertions.assertEquals("`${colName}`", params.get("subStringColName"));
+        params.clear();
 
-        OlapAnalysisTask olapAnalysisTask = new OlapAnalysisTask();
-        olapAnalysisTask.col = new Column("test", Type.fromPrimitiveType(PrimitiveType.STRING),
-            true, null, null, null);
-        olapAnalysisTask.tbl = tableIf;
-        Assertions.assertFalse(olapAnalysisTask.needLimit());
+        new MockUp<OlapAnalysisTask>() {
+            @Mock
+            protected boolean useLinearAnalyzeTemplate() {
+                return true;
+            }
 
-        olapAnalysisTask.col = new Column("test", Type.fromPrimitiveType(PrimitiveType.STRING),
-            false, null, null, null);
-        Assertions.assertFalse(olapAnalysisTask.needLimit());
-    }
+            @Mock
+            protected boolean isSingleUniqueKey() {
+                return false;
+            }
+        };
 
-    @Test
-    public void testNeedLimitTrue(@Mocked CatalogIf catalogIf, @Mocked DatabaseIf databaseIf, @Mocked OlapTable tableIf)
-            throws Exception {
+        task = new OlapAnalysisTask();
+        task.col = new Column("test", PrimitiveType.INT);
+        task.getSampleParams(params, 1000);
+        Assertions.assertEquals("10.0", params.get("scaleFactor"));
+        Assertions.assertEquals("TABLET(1, 2)", params.get("sampleHints"));
+        Assertions.assertEquals("ROUND(NDV(`${colName}`) * ${scaleFactor})", params.get("ndvFunction"));
+        params.clear();
 
-        new MockUp<OlapTable>() {
+        new MockUp<OlapAnalysisTask>() {
             @Mock
-            public PartitionInfo getPartitionInfo() {
-                ArrayList<Column> columns = Lists.newArrayList();
-                columns.add(new Column("NOFOUND", PrimitiveType.STRING));
-                return new PartitionInfo(PartitionType.RANGE, columns);
+            protected boolean isSingleUniqueKey() {
+                return true;
             }
         };
+        task = new OlapAnalysisTask();
+        task.col = new Column("test", PrimitiveType.INT);
+        task.getSampleParams(params, 1000);
+        Assertions.assertEquals("10.0", params.get("scaleFactor"));
+        Assertions.assertEquals("TABLET(1, 2)", params.get("sampleHints"));
+        Assertions.assertEquals("1000", params.get("ndvFunction"));
+        params.clear();
 
-        OlapAnalysisTask olapAnalysisTask = new OlapAnalysisTask();
-        olapAnalysisTask.tbl = tableIf;
-        olapAnalysisTask.col = new Column("test", Type.fromPrimitiveType(PrimitiveType.STRING),
-            false, null, null, null);
-        Assertions.assertTrue(olapAnalysisTask.needLimit());
-    }
+        new MockUp<OlapAnalysisTask>() {
+            @Mock
+            protected boolean needLimit() {
+                return true;
+            }
 
+            @Mock
+            protected long getSampleRows() {
+                return 50;
+            }
+        };
+        task = new OlapAnalysisTask();
+        task.col = new Column("test", PrimitiveType.INT);
+        task.getSampleParams(params, 1000);
+        Assertions.assertEquals("20.0", params.get("scaleFactor"));
+        Assertions.assertEquals("TABLET(1, 2)", params.get("sampleHints"));
+        Assertions.assertEquals("1000", params.get("ndvFunction"));
+        Assertions.assertEquals("limit 50", params.get("limit"));
+        params.clear();
+
+        task = new OlapAnalysisTask();
+        task.col = new Column("test", Type.fromPrimitiveType(PrimitiveType.INT),
+            true, null, null, null);
+        task.setKeyColumnSampleTooManyRows(true);
+        task.getSampleParams(params, 2000000000);
+        Assertions.assertEquals("2.0", params.get("scaleFactor"));
+        Assertions.assertEquals("TABLET(1, 2)", params.get("sampleHints"));
+        Assertions.assertEquals("2000000000", params.get("ndvFunction"));
+        Assertions.assertEquals("limit 1000000000", params.get("limit"));
+    }
 }

From 322de2b298cce71bd3a664ceb74e9a00a3a4bbad Mon Sep 17 00:00:00 2001
From: Pxl <xl@selectdb.com>
Date: Thu, 9 Jan 2025 11:46:04 +0800
Subject: [PATCH 064/140] [Enchancement](join) remove tuple is null maintain on
 join operators (#46618)

### What problem does this PR solve?
tuple_is_null is useless now
---
 .../pipeline/exec/hashjoin_probe_operator.cpp | 44 ---------------
 .../pipeline/exec/hashjoin_probe_operator.h   |  1 -
 .../exec/join/process_hash_table_probe.h      |  7 +--
 .../exec/join/process_hash_table_probe_impl.h | 53 +++----------------
 be/src/pipeline/exec/join_probe_operator.cpp  | 13 -----
 be/src/pipeline/exec/join_probe_operator.h    |  4 --
 .../exec/nested_loop_join_probe_operator.cpp  | 51 ------------------
 .../exec/nested_loop_join_probe_operator.h    |  2 -
 8 files changed, 8 insertions(+), 167 deletions(-)

diff --git a/be/src/pipeline/exec/hashjoin_probe_operator.cpp b/be/src/pipeline/exec/hashjoin_probe_operator.cpp
index 1f30a6183a22b96..8b7e8b8ea3aba51 100644
--- a/be/src/pipeline/exec/hashjoin_probe_operator.cpp
+++ b/be/src/pipeline/exec/hashjoin_probe_operator.cpp
@@ -144,8 +144,6 @@ Status HashJoinProbeLocalState::close(RuntimeState* state) {
     }
     _process_hashtable_ctx_variants = nullptr;
     _null_map_column = nullptr;
-    _tuple_is_null_left_flag_column = nullptr;
-    _tuple_is_null_right_flag_column = nullptr;
     _probe_block.clear();
     return JoinProbeLocalState<HashJoinSharedState, HashJoinProbeLocalState>::close(state);
 }
@@ -161,33 +159,6 @@ bool HashJoinProbeLocalState::_need_probe_null_map(vectorized::Block& block,
     return false;
 }
 
-void HashJoinProbeLocalState::add_tuple_is_null_column(vectorized::Block* block) {
-    DCHECK(_parent->cast<HashJoinProbeOperatorX>()._is_outer_join);
-    if (!_parent->cast<HashJoinProbeOperatorX>()._use_specific_projections) {
-        return;
-    }
-    auto p0 = _tuple_is_null_left_flag_column->assume_mutable();
-    auto p1 = _tuple_is_null_right_flag_column->assume_mutable();
-    auto& left_null_map = reinterpret_cast<vectorized::ColumnUInt8&>(*p0);
-    auto& right_null_map = reinterpret_cast<vectorized::ColumnUInt8&>(*p1);
-    auto left_size = left_null_map.size();
-    auto right_size = right_null_map.size();
-
-    if (left_size == 0) {
-        DCHECK_EQ(right_size, block->rows());
-        left_null_map.get_data().resize_fill(right_size, 0);
-    }
-    if (right_size == 0) {
-        DCHECK_EQ(left_size, block->rows());
-        right_null_map.get_data().resize_fill(left_size, 0);
-    }
-
-    block->insert(
-            {std::move(p0), std::make_shared<vectorized::DataTypeUInt8>(), "left_tuples_is_null"});
-    block->insert(
-            {std::move(p1), std::make_shared<vectorized::DataTypeUInt8>(), "right_tuples_is_null"});
-}
-
 void HashJoinProbeLocalState::_prepare_probe_block() {
     // clear_column_data of _probe_block
     if (!_probe_column_disguise_null.empty()) {
@@ -258,16 +229,6 @@ Status HashJoinProbeOperatorX::pull(doris::RuntimeState* state, vectorized::Bloc
             local_state._probe_block.insert({std::move(nullable_column), make_nullable(type),
                                              _right_table_column_names[i]});
         }
-        if (_is_outer_join) {
-            reinterpret_cast<vectorized::ColumnUInt8*>(
-                    local_state._tuple_is_null_left_flag_column.get())
-                    ->get_data()
-                    .resize_fill(block_rows, 0);
-            reinterpret_cast<vectorized::ColumnUInt8*>(
-                    local_state._tuple_is_null_right_flag_column.get())
-                    ->get_data()
-                    .resize_fill(block_rows, 1);
-        }
 
         /// No need to check the block size in `_filter_data_and_build_output` because here dose not
         /// increase the output rows count(just same as `_probe_block`'s rows count).
@@ -414,10 +375,6 @@ Status HashJoinProbeLocalState::filter_data_and_build_output(RuntimeState* state
                                                              bool* eos,
                                                              vectorized::Block* temp_block,
                                                              bool check_rows_count) {
-    auto& p = _parent->cast<HashJoinProbeOperatorX>();
-    if (p._is_outer_join) {
-        add_tuple_is_null_column(temp_block);
-    }
     auto output_rows = temp_block->rows();
     if (check_rows_count) {
         DCHECK(output_rows <= state->batch_size());
@@ -429,7 +386,6 @@ Status HashJoinProbeLocalState::filter_data_and_build_output(RuntimeState* state
     }
 
     RETURN_IF_ERROR(_build_output_block(temp_block, output_block, false));
-    _reset_tuple_is_null_column();
     reached_limit(output_block, eos);
     return Status::OK();
 }
diff --git a/be/src/pipeline/exec/hashjoin_probe_operator.h b/be/src/pipeline/exec/hashjoin_probe_operator.h
index 533aaefa5beef9d..a632bd81bbadfbe 100644
--- a/be/src/pipeline/exec/hashjoin_probe_operator.h
+++ b/be/src/pipeline/exec/hashjoin_probe_operator.h
@@ -56,7 +56,6 @@ class HashJoinProbeLocalState final
     Status close(RuntimeState* state) override;
 
     void prepare_for_next();
-    void add_tuple_is_null_column(vectorized::Block* block) override;
     Status filter_data_and_build_output(RuntimeState* state, vectorized::Block* output_block,
                                         bool* eos, vectorized::Block* temp_block,
                                         bool check_rows_count = true);
diff --git a/be/src/pipeline/exec/join/process_hash_table_probe.h b/be/src/pipeline/exec/join/process_hash_table_probe.h
index 052ed5875ae2965..ae7a3261bb1a016 100644
--- a/be/src/pipeline/exec/join/process_hash_table_probe.h
+++ b/be/src/pipeline/exec/join/process_hash_table_probe.h
@@ -52,8 +52,7 @@ struct ProcessHashTableProbe {
 
     void probe_side_output_column(vectorized::MutableColumns& mcol,
                                   const std::vector<bool>& output_slot_flags, int size,
-                                  int last_probe_index, bool all_match_one,
-                                  bool have_other_join_conjunct);
+                                  bool all_match_one, bool have_other_join_conjunct);
 
     template <typename HashTableType>
     Status process(HashTableType& hash_table_ctx, ConstNullMapPtr null_map,
@@ -111,10 +110,6 @@ struct ProcessHashTableProbe {
     uint32_t _build_index_for_null_probe_key {0};
 
     std::vector<int> _build_blocks_locs;
-    // only need set the tuple is null in RIGHT_OUTER_JOIN and FULL_OUTER_JOIN
-    vectorized::ColumnUInt8::Container* _tuple_is_null_left_flags = nullptr;
-    // only need set the tuple is null in LEFT_OUTER_JOIN and FULL_OUTER_JOIN
-    vectorized::ColumnUInt8::Container* _tuple_is_null_right_flags = nullptr;
 
     size_t _serialized_key_buffer_size {0};
     uint8_t* _serialized_key_buffer = nullptr;
diff --git a/be/src/pipeline/exec/join/process_hash_table_probe_impl.h b/be/src/pipeline/exec/join/process_hash_table_probe_impl.h
index eb4a36d17cfd003..8066e28275e34ab 100644
--- a/be/src/pipeline/exec/join/process_hash_table_probe_impl.h
+++ b/be/src/pipeline/exec/join/process_hash_table_probe_impl.h
@@ -39,16 +39,6 @@ ProcessHashTableProbe<JoinOpType>::ProcessHashTableProbe(HashJoinProbeLocalState
         : _parent(parent),
           _batch_size(batch_size),
           _build_block(parent->build_block()),
-          _tuple_is_null_left_flags(parent->is_outer_join()
-                                            ? &(reinterpret_cast<vectorized::ColumnUInt8&>(
-                                                        *parent->_tuple_is_null_left_flag_column)
-                                                        .get_data())
-                                            : nullptr),
-          _tuple_is_null_right_flags(parent->is_outer_join()
-                                             ? &(reinterpret_cast<vectorized::ColumnUInt8&>(
-                                                         *parent->_tuple_is_null_right_flag_column)
-                                                         .get_data())
-                                             : nullptr),
           _have_other_join_conjunct(parent->have_other_join_conjunct()),
           _is_right_semi_anti(parent->is_right_semi_anti()),
           _left_output_slot_flags(parent->left_output_slot_flags()),
@@ -70,9 +60,6 @@ void ProcessHashTableProbe<JoinOpType>::build_side_output_column(
         bool have_other_join_conjunct, bool is_mark_join) {
     SCOPED_TIMER(_build_side_output_timer);
 
-    constexpr auto probe_all =
-            JoinOpType == TJoinOp::LEFT_OUTER_JOIN || JoinOpType == TJoinOp::FULL_OUTER_JOIN;
-
     // indicates whether build_indexs contain 0
     bool build_index_has_zero =
             (JoinOpType != TJoinOp::INNER_JOIN && JoinOpType != TJoinOp::RIGHT_OUTER_JOIN) ||
@@ -80,17 +67,6 @@ void ProcessHashTableProbe<JoinOpType>::build_side_output_column(
     if (!size) {
         return;
     }
-    // Dispose right tuple is null flags columns
-    if (probe_all && !have_other_join_conjunct) {
-        _tuple_is_null_right_flags->resize(size);
-        auto* __restrict null_data = _tuple_is_null_right_flags->data();
-        for (int i = 0; i < size; ++i) {
-            null_data[i] = _build_indexs[i] == 0;
-        }
-        if (_need_calculate_build_index_has_zero) {
-            build_index_has_zero = simd::contain_byte(null_data, size, 1);
-        }
-    }
 
     if (!build_index_has_zero && _build_column_has_null.empty()) {
         _need_calculate_build_index_has_zero = false;
@@ -141,14 +117,14 @@ void ProcessHashTableProbe<JoinOpType>::build_side_output_column(
 template <int JoinOpType>
 void ProcessHashTableProbe<JoinOpType>::probe_side_output_column(
         vectorized::MutableColumns& mcol, const std::vector<bool>& output_slot_flags, int size,
-        int last_probe_index, bool all_match_one, bool have_other_join_conjunct) {
+        bool all_match_one, bool have_other_join_conjunct) {
     SCOPED_TIMER(_probe_side_output_timer);
     auto& probe_block = _parent->_probe_block;
     for (int i = 0; i < output_slot_flags.size(); ++i) {
         if (output_slot_flags[i]) {
             auto& column = probe_block.get_by_position(i).column;
             if (all_match_one) {
-                mcol[i]->insert_range_from(*column, last_probe_index, size);
+                mcol[i]->insert_range_from(*column, _probe_indexs[0], size);
             } else {
                 mcol[i]->insert_indices_from(*column, _probe_indexs.data(),
                                              _probe_indexs.data() + size);
@@ -158,12 +134,6 @@ void ProcessHashTableProbe<JoinOpType>::probe_side_output_column(
             mcol[i] = vectorized::ColumnConst::create(std::move(mcol[i]), size);
         }
     }
-
-    if constexpr (JoinOpType == TJoinOp::RIGHT_OUTER_JOIN) {
-        if (!have_other_join_conjunct) {
-            _tuple_is_null_left_flags->resize_fill(size, 0);
-        }
-    }
 }
 
 template <int JoinOpType>
@@ -212,7 +182,6 @@ Status ProcessHashTableProbe<JoinOpType>::do_process(HashTableType& hash_table_c
 
     auto& probe_index = _parent->_probe_index;
     auto& build_index = _parent->_build_index;
-    auto last_probe_index = probe_index;
     {
         SCOPED_TIMER(_init_probe_side_timer);
         _init_probe_side<HashTableType>(hash_table_ctx, probe_rows, with_other_conjuncts, null_map);
@@ -277,9 +246,8 @@ Status ProcessHashTableProbe<JoinOpType>::do_process(HashTableType& hash_table_c
 
     if constexpr (with_other_conjuncts || (JoinOpType != TJoinOp::RIGHT_SEMI_JOIN &&
                                            JoinOpType != TJoinOp::RIGHT_ANTI_JOIN)) {
-        auto check_all_match_one = [](const std::vector<uint32_t>& vecs, uint32_t probe_idx,
-                                      int size) {
-            if (!size || vecs[0] != probe_idx || vecs[size - 1] != probe_idx + size - 1) {
+        auto check_all_match_one = [](const std::vector<uint32_t>& vecs, int size) {
+            if (!size || vecs[size - 1] != vecs[0] + size - 1) {
                 return false;
             }
             for (int i = 1; i < size; i++) {
@@ -290,10 +258,9 @@ Status ProcessHashTableProbe<JoinOpType>::do_process(HashTableType& hash_table_c
             return true;
         };
 
-        probe_side_output_column(
-                mcol, *_left_output_slot_flags, current_offset, last_probe_index,
-                check_all_match_one(_probe_indexs, last_probe_index, current_offset),
-                with_other_conjuncts);
+        probe_side_output_column(mcol, *_left_output_slot_flags, current_offset,
+                                 check_all_match_one(_probe_indexs, current_offset),
+                                 with_other_conjuncts);
     }
 
     output_block->swap(mutable_block.to_block());
@@ -540,8 +507,6 @@ Status ProcessHashTableProbe<JoinOpType>::do_other_join_conjuncts(vectorized::Bl
 
         for (size_t i = 0; i < row_count; ++i) {
             if (filter_map[i]) {
-                _tuple_is_null_right_flags->emplace_back(!_build_indexs[i] ||
-                                                         !filter_column_ptr[i]);
                 if constexpr (JoinOpType == TJoinOp::FULL_OUTER_JOIN) {
                     visited[_build_indexs[i]] = 1;
                 }
@@ -585,12 +550,9 @@ Status ProcessHashTableProbe<JoinOpType>::do_other_join_conjuncts(vectorized::Bl
             visited[_build_indexs[i]] |= filter_column_ptr[i];
         }
     } else if constexpr (JoinOpType == TJoinOp::RIGHT_OUTER_JOIN) {
-        auto filter_size = 0;
         for (int i = 0; i < row_count; ++i) {
             visited[_build_indexs[i]] |= filter_column_ptr[i];
-            filter_size += filter_column_ptr[i];
         }
-        _tuple_is_null_left_flags->resize_fill(filter_size, 0);
     }
 
     if constexpr (JoinOpType == TJoinOp::RIGHT_SEMI_JOIN ||
@@ -659,7 +621,6 @@ Status ProcessHashTableProbe<JoinOpType>::finish_probing(HashTableType& hash_tab
                 assert_cast<vectorized::ColumnNullable*>(mcol[i].get())
                         ->insert_many_defaults(block_size);
             }
-            _tuple_is_null_left_flags->resize_fill(block_size, 1);
         }
         output_block->swap(mutable_block.to_block(0));
         DCHECK(block_size <= _batch_size);
diff --git a/be/src/pipeline/exec/join_probe_operator.cpp b/be/src/pipeline/exec/join_probe_operator.cpp
index 2ebe3a9d0557660..5249e9849aa2e25 100644
--- a/be/src/pipeline/exec/join_probe_operator.cpp
+++ b/be/src/pipeline/exec/join_probe_operator.cpp
@@ -42,11 +42,6 @@ template <typename SharedStateArg, typename Derived>
 Status JoinProbeLocalState<SharedStateArg, Derived>::open(RuntimeState* state) {
     RETURN_IF_ERROR(Base::open(state));
     auto& p = Base::_parent->template cast<typename Derived::Parent>();
-    // only use in outer join as the bool column to mark for function of `tuple_is_null`
-    if (p._is_outer_join) {
-        _tuple_is_null_left_flag_column = vectorized::ColumnUInt8::create();
-        _tuple_is_null_right_flag_column = vectorized::ColumnUInt8::create();
-    }
     _output_expr_ctxs.resize(p._output_expr_ctxs.size());
     for (size_t i = 0; i < _output_expr_ctxs.size(); i++) {
         RETURN_IF_ERROR(p._output_expr_ctxs[i]->clone(state, _output_expr_ctxs[i]));
@@ -96,14 +91,6 @@ Status JoinProbeLocalState<SharedStateArg, Derived>::_build_output_block(
     return Status::OK();
 }
 
-template <typename SharedStateArg, typename Derived>
-void JoinProbeLocalState<SharedStateArg, Derived>::_reset_tuple_is_null_column() {
-    if (Base::_parent->template cast<typename Derived::Parent>()._is_outer_join) {
-        reinterpret_cast<vectorized::ColumnUInt8&>(*_tuple_is_null_left_flag_column).clear();
-        reinterpret_cast<vectorized::ColumnUInt8&>(*_tuple_is_null_right_flag_column).clear();
-    }
-}
-
 template <typename LocalStateType>
 JoinProbeOperatorX<LocalStateType>::JoinProbeOperatorX(ObjectPool* pool, const TPlanNode& tnode,
                                                        int operator_id, const DescriptorTbl& descs)
diff --git a/be/src/pipeline/exec/join_probe_operator.h b/be/src/pipeline/exec/join_probe_operator.h
index 161fd18fa1dab84..e5722dd6456c8ec 100644
--- a/be/src/pipeline/exec/join_probe_operator.h
+++ b/be/src/pipeline/exec/join_probe_operator.h
@@ -30,7 +30,6 @@ class JoinProbeLocalState : public PipelineXLocalState<SharedStateArg> {
     Status init(RuntimeState* state, LocalStateInfo& info) override;
     Status open(RuntimeState* state) override;
     Status close(RuntimeState* state) override;
-    virtual void add_tuple_is_null_column(vectorized::Block* block) = 0;
 
 protected:
     template <typename LocalStateType>
@@ -41,12 +40,9 @@ class JoinProbeLocalState : public PipelineXLocalState<SharedStateArg> {
     void _construct_mutable_join_block();
     Status _build_output_block(vectorized::Block* origin_block, vectorized::Block* output_block,
                                bool keep_origin = true);
-    void _reset_tuple_is_null_column();
     // output expr
     vectorized::VExprContextSPtrs _output_expr_ctxs;
     vectorized::Block _join_block;
-    vectorized::MutableColumnPtr _tuple_is_null_left_flag_column = nullptr;
-    vectorized::MutableColumnPtr _tuple_is_null_right_flag_column = nullptr;
 
     size_t _mark_column_id = -1;
 
diff --git a/be/src/pipeline/exec/nested_loop_join_probe_operator.cpp b/be/src/pipeline/exec/nested_loop_join_probe_operator.cpp
index e2c6b7d84f02f85..4d479371e95db32 100644
--- a/be/src/pipeline/exec/nested_loop_join_probe_operator.cpp
+++ b/be/src/pipeline/exec/nested_loop_join_probe_operator.cpp
@@ -71,29 +71,12 @@ Status NestedLoopJoinProbeLocalState::close(RuntimeState* state) {
     }
     _child_block->clear();
 
-    _tuple_is_null_left_flag_column = nullptr;
-    _tuple_is_null_right_flag_column = nullptr;
     return JoinProbeLocalState<NestedLoopJoinSharedState, NestedLoopJoinProbeLocalState>::close(
             state);
 }
 
 void NestedLoopJoinProbeLocalState::_update_additional_flags(vectorized::Block* block) {
     auto& p = _parent->cast<NestedLoopJoinProbeOperatorX>();
-    if (p._is_outer_join) {
-        auto p0 = _tuple_is_null_left_flag_column->assume_mutable();
-        auto p1 = _tuple_is_null_right_flag_column->assume_mutable();
-        auto& left_null_map = reinterpret_cast<vectorized::ColumnUInt8&>(*p0);
-        auto& right_null_map = reinterpret_cast<vectorized::ColumnUInt8&>(*p1);
-        auto left_size = left_null_map.size();
-        auto right_size = right_null_map.size();
-
-        if (left_size < block->rows()) {
-            left_null_map.get_data().resize_fill(block->rows(), 0);
-        }
-        if (right_size < block->rows()) {
-            right_null_map.get_data().resize_fill(block->rows(), 0);
-        }
-    }
     if (p._is_mark_join) {
         auto mark_column = block->get_by_position(block->columns() - 1).column->assume_mutable();
         if (mark_column->size() < block->rows()) {
@@ -108,21 +91,6 @@ void NestedLoopJoinProbeLocalState::_reset_with_next_probe_row() {
     _left_block_pos++;
 }
 
-void NestedLoopJoinProbeLocalState::add_tuple_is_null_column(vectorized::Block* block) {
-    auto& p = _parent->cast<NestedLoopJoinProbeOperatorX>();
-    if (!p._use_specific_projections) {
-        return;
-    }
-    if (p._is_outer_join) {
-        auto p0 = _tuple_is_null_left_flag_column->assume_mutable();
-        auto p1 = _tuple_is_null_right_flag_column->assume_mutable();
-        block->insert({std::move(p0), std::make_shared<vectorized::DataTypeUInt8>(),
-                       "left_tuples_is_null"});
-        block->insert({std::move(p1), std::make_shared<vectorized::DataTypeUInt8>(),
-                       "right_tuples_is_null"});
-    }
-}
-
 template <typename JoinOpType, bool set_build_side_flag, bool set_probe_side_flag>
 Status NestedLoopJoinProbeLocalState::generate_join_block_data(RuntimeState* state,
                                                                JoinOpType& join_op_variants) {
@@ -212,20 +180,6 @@ Status NestedLoopJoinProbeLocalState::generate_join_block_data(RuntimeState* sta
     return Status::OK();
 }
 
-void NestedLoopJoinProbeLocalState::_resize_fill_tuple_is_null_column(size_t new_size,
-                                                                      uint8_t left_flag,
-                                                                      uint8_t right_flag) {
-    auto& p = _parent->cast<NestedLoopJoinProbeOperatorX>();
-    if (p._is_outer_join) {
-        reinterpret_cast<vectorized::ColumnUInt8*>(_tuple_is_null_left_flag_column.get())
-                ->get_data()
-                .resize_fill(new_size, left_flag);
-        reinterpret_cast<vectorized::ColumnUInt8*>(_tuple_is_null_right_flag_column.get())
-                ->get_data()
-                .resize_fill(new_size, right_flag);
-    }
-}
-
 template <bool BuildSide, bool IsSemi>
 void NestedLoopJoinProbeLocalState::_finalize_current_phase(vectorized::Block& block,
                                                             size_t batch_size) {
@@ -289,7 +243,6 @@ void NestedLoopJoinProbeLocalState::_finalize_current_phase(vectorized::Block& b
                             selector.data() + selector_idx);
                 }
             }
-            _resize_fill_tuple_is_null_column(column_size, 1, 0);
         }
         _output_null_idx_build_side = i;
     } else {
@@ -323,7 +276,6 @@ void NestedLoopJoinProbeLocalState::_finalize_current_phase(vectorized::Block& b
                     dst_columns[p._num_probe_side_columns + i]->insert_many_defaults(new_size -
                                                                                      column_size);
                 }
-                _resize_fill_tuple_is_null_column(new_size, 0, 1);
             }
         } else {
             vectorized::ColumnFilterHelper mark_column(*dst_columns[dst_columns.size() - 1]);
@@ -344,7 +296,6 @@ void NestedLoopJoinProbeLocalState::_finalize_current_phase(vectorized::Block& b
                 dst_columns[p._num_probe_side_columns + i]->insert_many_defaults(
                         _left_side_process_count);
             }
-            _resize_fill_tuple_is_null_column(_left_side_process_count, 0, 1);
         }
     }
     block.set_columns(std::move(dst_columns));
@@ -519,7 +470,6 @@ Status NestedLoopJoinProbeOperatorX::pull(RuntimeState* state, vectorized::Block
 
         size_t join_block_column_size = local_state._join_block.columns();
         {
-            local_state.add_tuple_is_null_column(&local_state._join_block);
             {
                 SCOPED_TIMER(local_state._join_filter_timer);
                 RETURN_IF_ERROR(vectorized::VExprContext::filter_block(
@@ -527,7 +477,6 @@ Status NestedLoopJoinProbeOperatorX::pull(RuntimeState* state, vectorized::Block
                         local_state._join_block.columns()));
             }
             RETURN_IF_ERROR(local_state._build_output_block(&local_state._join_block, block));
-            local_state._reset_tuple_is_null_column();
         }
         local_state._join_block.clear_column_data(join_block_column_size);
         if (!(*eos) and !local_state._need_more_input_data) {
diff --git a/be/src/pipeline/exec/nested_loop_join_probe_operator.h b/be/src/pipeline/exec/nested_loop_join_probe_operator.h
index c744e6acdc507e3..4f50a98efdb1fb7 100644
--- a/be/src/pipeline/exec/nested_loop_join_probe_operator.h
+++ b/be/src/pipeline/exec/nested_loop_join_probe_operator.h
@@ -41,7 +41,6 @@ class NestedLoopJoinProbeLocalState final
     NestedLoopJoinProbeLocalState(RuntimeState* state, OperatorXBase* parent);
     ~NestedLoopJoinProbeLocalState() = default;
 
-    void add_tuple_is_null_column(vectorized::Block* block) override;
 #define CLEAR_BLOCK                                                  \
     for (size_t i = 0; i < column_to_keep; ++i) {                    \
         block->get_by_position(i).column->assume_mutable()->clear(); \
@@ -57,7 +56,6 @@ class NestedLoopJoinProbeLocalState final
     void _update_additional_flags(vectorized::Block* block);
     template <bool BuildSide, bool IsSemi>
     void _finalize_current_phase(vectorized::Block& block, size_t batch_size);
-    void _resize_fill_tuple_is_null_column(size_t new_size, uint8_t left_flag, uint8_t right_flag);
     void _reset_with_next_probe_row();
     void _append_left_data_with_null(vectorized::Block& block) const;
     void _process_left_child_block(vectorized::Block& block,

From bbf8a819de19d36a31aa057e6f10c60c2466eb58 Mon Sep 17 00:00:00 2001
From: yujun <yujun@selectdb.com>
Date: Thu, 9 Jan 2025 12:06:45 +0800
Subject: [PATCH 065/140] [opt](nereids) opt range inference for or expression
 when out of order (#46303)

### What problem does this PR solve?

Problem Summary:

For range inference, it will merge multiple value desc whose reference
are the same. It will merge two value desc step by step. Diff merge
order may get diff result.

For range Inference: `x1 op x2 op x3 op x4`

If op is `AND`, then the merge order doesn't matter. It will always get
the same result.

But if op is `OR`, then the merge order does matter. For example: `(a <
10) or ( a > 30) or (a >= 15 and a <= 35)`. When merge the first OP, it
will get an UnknownValue: and its source is: `[ (-00, 10), (30, +00) ]`,
latter will merge this UnknowValue with RangeValue `[15, 35]`. Since
UnknowValue union another value desc will get a new UnknownValue, then
then final result is UknownValue(UnknowValue(RangeValue(`a<10`) or
RangeValue(`a>30`)) or RangeValue(`a>=15 and a <= 35`)). This is bad. It
should merge the 1st and 3rd value desc firstly, latter merge the 2nd
value desc, Then finally the merge result is 'TRUE'.

In order to achieve this, use a RangeSet to record all the ranges, then
RangeSet will auto merge the results.

What's more, this pr also:
1.  opt  'a > 20 or a = 20'  to  'a >= 20';
2. for the discrete value's options, if an option is in one range, then
the option will eliminate. for example: `a <= 10 or a in [1, 2, 3, 11,
12, 13]` will opt to `a <= 10 or a in [11, 12, 13]`;
3. delete toExpr in RangeInference;
---
 .../expression/rules/RangeInference.java      | 197 ++++++++++--------
 .../rules/expression/rules/SimplifyRange.java |  23 +-
 .../rules/expression/SimplifyRangeTest.java   |  63 ++++--
 .../doris/nereids/sqltest/InferTest.java      |   4 +-
 4 files changed, 171 insertions(+), 116 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RangeInference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RangeInference.java
index 247856578c29ffa..c78ec7a75fbad18 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RangeInference.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RangeInference.java
@@ -34,15 +34,17 @@
 import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
 import org.apache.doris.nereids.util.ExpressionUtils;
 
+import com.google.common.collect.BoundType;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Multimap;
 import com.google.common.collect.Multimaps;
 import com.google.common.collect.Range;
+import com.google.common.collect.RangeSet;
 import com.google.common.collect.Sets;
+import com.google.common.collect.TreeRangeSet;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -118,18 +120,17 @@ public ValueDesc visitInPredicate(InPredicate inPredicate, ExpressionRewriteCont
 
     @Override
     public ValueDesc visitAnd(And and, ExpressionRewriteContext context) {
-        return simplify(context, and, ExpressionUtils.extractConjunction(and),
+        return simplify(context, ExpressionUtils.extractConjunction(and),
                 ValueDesc::intersect, true);
     }
 
     @Override
     public ValueDesc visitOr(Or or, ExpressionRewriteContext context) {
-        return simplify(context, or, ExpressionUtils.extractDisjunction(or),
+        return simplify(context, ExpressionUtils.extractDisjunction(or),
                 ValueDesc::union, false);
     }
 
-    private ValueDesc simplify(ExpressionRewriteContext context,
-            Expression originExpr, List<Expression> predicates,
+    private ValueDesc simplify(ExpressionRewriteContext context, List<Expression> predicates,
             BinaryOperator<ValueDesc> op, boolean isAnd) {
 
         boolean convertIsNullToEmptyValue = isAnd && predicates.stream().anyMatch(expr -> expr instanceof NullLiteral);
@@ -144,7 +145,7 @@ private ValueDesc simplify(ExpressionRewriteContext context,
             // but we don't consider this case here, we should fold IsNull(a) to FALSE using other rule.
             ValueDesc valueDesc = null;
             if (convertIsNullToEmptyValue && predicate instanceof IsNull) {
-                valueDesc = new EmptyValue(context, ((IsNull) predicate).child(), predicate);
+                valueDesc = new EmptyValue(context, ((IsNull) predicate).child());
             } else {
                 valueDesc = predicate.accept(this, context);
             }
@@ -154,7 +155,11 @@ private ValueDesc simplify(ExpressionRewriteContext context,
 
         List<ValueDesc> valuePerRefs = Lists.newArrayList();
         for (Entry<Expression, Collection<ValueDesc>> referenceValues : groupByReference.asMap().entrySet()) {
+            Expression reference = referenceValues.getKey();
             List<ValueDesc> valuePerReference = (List) referenceValues.getValue();
+            if (!isAnd) {
+                valuePerReference = ValueDesc.unionDiscreteAndRange(context, reference, valuePerReference);
+            }
 
             // merge per reference
             ValueDesc simplifiedValue = valuePerReference.get(0);
@@ -170,7 +175,7 @@ private ValueDesc simplify(ExpressionRewriteContext context,
         }
 
         // use UnknownValue to wrap different references
-        return new UnknownValue(context, originExpr, valuePerRefs, isAnd);
+        return new UnknownValue(context, valuePerRefs, isAnd);
     }
 
     /**
@@ -178,12 +183,10 @@ private ValueDesc simplify(ExpressionRewriteContext context,
      */
     public abstract static class ValueDesc {
         ExpressionRewriteContext context;
-        Expression toExpr;
         Expression reference;
 
-        public ValueDesc(ExpressionRewriteContext context, Expression reference, Expression toExpr) {
+        public ValueDesc(ExpressionRewriteContext context, Expression reference) {
             this.context = context;
-            this.toExpr = toExpr;
             this.reference = reference;
         }
 
@@ -191,10 +194,6 @@ public Expression getReference() {
             return reference;
         }
 
-        public Expression getOriginExpr() {
-            return toExpr;
-        }
-
         public ExpressionRewriteContext getExpressionRewriteContext() {
             return context;
         }
@@ -204,16 +203,62 @@ public ExpressionRewriteContext getExpressionRewriteContext() {
         /** or */
         public static ValueDesc union(ExpressionRewriteContext context,
                 RangeValue range, DiscreteValue discrete, boolean reverseOrder) {
-            long count = discrete.values.stream().filter(x -> range.range.test(x)).count();
-            if (count == discrete.values.size()) {
+            if (discrete.values.stream().allMatch(x -> range.range.test(x))) {
                 return range;
             }
-            Expression toExpr = FoldConstantRuleOnFE.evaluate(
-                    new Or(range.toExpr, discrete.toExpr), context);
             List<ValueDesc> sourceValues = reverseOrder
                     ? ImmutableList.of(discrete, range)
                     : ImmutableList.of(range, discrete);
-            return new UnknownValue(context, toExpr, sourceValues, false);
+            return new UnknownValue(context, sourceValues, false);
+        }
+
+        /** merge discrete and ranges only, no merge other value desc */
+        public static List<ValueDesc> unionDiscreteAndRange(ExpressionRewriteContext context,
+                Expression reference, List<ValueDesc> valueDescs) {
+            Set<Literal> discreteValues = Sets.newHashSet();
+            for (ValueDesc valueDesc : valueDescs) {
+                if (valueDesc instanceof DiscreteValue) {
+                    discreteValues.addAll(((DiscreteValue) valueDesc).getValues());
+                }
+            }
+
+            // for 'a > 8 or a = 8', then range (8, +00) can convert to [8, +00)
+            RangeSet<Literal> rangeSet = TreeRangeSet.create();
+            for (ValueDesc valueDesc : valueDescs) {
+                if (valueDesc instanceof RangeValue) {
+                    Range<Literal> range = ((RangeValue) valueDesc).range;
+                    rangeSet.add(range);
+                    if (range.hasLowerBound()
+                            && range.lowerBoundType() == BoundType.OPEN
+                            && discreteValues.contains(range.lowerEndpoint())) {
+                        rangeSet.add(Range.singleton(range.lowerEndpoint()));
+                    }
+                    if (range.hasUpperBound()
+                            && range.upperBoundType() == BoundType.OPEN
+                            && discreteValues.contains(range.upperEndpoint())) {
+                        rangeSet.add(Range.singleton(range.upperEndpoint()));
+                    }
+                }
+            }
+
+            if (!rangeSet.isEmpty()) {
+                discreteValues.removeIf(x -> rangeSet.contains(x));
+            }
+
+            List<ValueDesc> result = Lists.newArrayListWithExpectedSize(valueDescs.size());
+            if (!discreteValues.isEmpty()) {
+                result.add(new DiscreteValue(context, reference, discreteValues));
+            }
+            for (Range<Literal> range : rangeSet.asRanges()) {
+                result.add(new RangeValue(context, reference, range));
+            }
+            for (ValueDesc valueDesc : valueDescs) {
+                if (!(valueDesc instanceof DiscreteValue) && !(valueDesc instanceof RangeValue)) {
+                    result.add(valueDesc);
+                }
+            }
+
+            return result;
         }
 
         /** intersect */
@@ -221,19 +266,19 @@ public static ValueDesc union(ExpressionRewriteContext context,
 
         /** intersect */
         public static ValueDesc intersect(ExpressionRewriteContext context, RangeValue range, DiscreteValue discrete) {
-            DiscreteValue result = new DiscreteValue(context, discrete.reference, discrete.toExpr);
-            discrete.values.stream().filter(x -> range.range.contains(x)).forEach(result.values::add);
-            if (!result.values.isEmpty()) {
-                return result;
+            Set<Literal> newValues = discrete.values.stream().filter(x -> range.range.contains(x))
+                    .collect(Collectors.toSet());
+            if (newValues.isEmpty()) {
+                return new EmptyValue(context, range.reference);
+            } else {
+                return new DiscreteValue(context, range.reference, newValues);
             }
-            Expression originExpr = FoldConstantRuleOnFE.evaluate(new And(range.toExpr, discrete.toExpr), context);
-            return new EmptyValue(context, range.reference, originExpr);
         }
 
         private static ValueDesc range(ExpressionRewriteContext context, ComparisonPredicate predicate) {
             Literal value = (Literal) predicate.right();
             if (predicate instanceof EqualTo) {
-                return new DiscreteValue(context, predicate.left(), predicate, value);
+                return new DiscreteValue(context, predicate.left(), Sets.newHashSet(value));
             }
             Range<Literal> range = null;
             if (predicate instanceof GreaterThanEqual) {
@@ -246,13 +291,13 @@ private static ValueDesc range(ExpressionRewriteContext context, ComparisonPredi
                 range = Range.lessThan(value);
             }
 
-            return new RangeValue(context, predicate.left(), predicate, range);
+            return new RangeValue(context, predicate.left(), range);
         }
 
         public static ValueDesc discrete(ExpressionRewriteContext context, InPredicate in) {
             // Set<Literal> literals = (Set) Utils.fastToImmutableSet(in.getOptions());
             Set<Literal> literals = in.getOptions().stream().map(Literal.class::cast).collect(Collectors.toSet());
-            return new DiscreteValue(context, in.getCompareExpr(), in, literals);
+            return new DiscreteValue(context, in.getCompareExpr(), literals);
         }
     }
 
@@ -261,8 +306,8 @@ public static ValueDesc discrete(ExpressionRewriteContext context, InPredicate i
      */
     public static class EmptyValue extends ValueDesc {
 
-        public EmptyValue(ExpressionRewriteContext context, Expression reference, Expression toExpr) {
-            super(context, reference, toExpr);
+        public EmptyValue(ExpressionRewriteContext context, Expression reference) {
+            super(context, reference);
         }
 
         @Override
@@ -284,9 +329,8 @@ public ValueDesc intersect(ValueDesc other) {
     public static class RangeValue extends ValueDesc {
         Range<Literal> range;
 
-        public RangeValue(ExpressionRewriteContext context, Expression reference,
-                Expression toExpr, Range<Literal> range) {
-            super(context, reference, toExpr);
+        public RangeValue(ExpressionRewriteContext context, Expression reference, Range<Literal> range) {
+            super(context, reference);
             this.range = range;
         }
 
@@ -300,20 +344,16 @@ public ValueDesc union(ValueDesc other) {
                 return other.union(this);
             }
             if (other instanceof RangeValue) {
-                Expression originExpr = FoldConstantRuleOnFE.evaluate(new Or(toExpr, other.toExpr), context);
                 RangeValue o = (RangeValue) other;
                 if (range.isConnected(o.range)) {
-                    return new RangeValue(context, reference, originExpr, range.span(o.range));
+                    return new RangeValue(context, reference, range.span(o.range));
                 }
-                return new UnknownValue(context, originExpr,
-                        ImmutableList.of(this, other), false);
+                return new UnknownValue(context, ImmutableList.of(this, other), false);
             }
             if (other instanceof DiscreteValue) {
                 return union(context, this, (DiscreteValue) other, false);
             }
-            Expression originExpr = FoldConstantRuleOnFE.evaluate(new Or(toExpr, other.toExpr), context);
-            return new UnknownValue(context, originExpr,
-                    ImmutableList.of(this, other), false);
+            return new UnknownValue(context, ImmutableList.of(this, other), false);
         }
 
         @Override
@@ -322,19 +362,16 @@ public ValueDesc intersect(ValueDesc other) {
                 return other.intersect(this);
             }
             if (other instanceof RangeValue) {
-                Expression originExpr = FoldConstantRuleOnFE.evaluate(new And(toExpr, other.toExpr), context);
                 RangeValue o = (RangeValue) other;
                 if (range.isConnected(o.range)) {
-                    return new RangeValue(context, reference, originExpr, range.intersection(o.range));
+                    return new RangeValue(context, reference, range.intersection(o.range));
                 }
-                return new EmptyValue(context, reference, originExpr);
+                return new EmptyValue(context, reference);
             }
             if (other instanceof DiscreteValue) {
                 return intersect(context, this, (DiscreteValue) other);
             }
-            Expression originExpr = FoldConstantRuleOnFE.evaluate(new And(toExpr, other.toExpr), context);
-            return new UnknownValue(context, originExpr,
-                    ImmutableList.of(this, other), true);
+            return new UnknownValue(context, ImmutableList.of(this, other), true);
         }
 
         @Override
@@ -349,17 +386,12 @@ public String toString() {
      * a in (1,2,3) => [1,2,3]
      */
     public static class DiscreteValue extends ValueDesc {
-        Set<Literal> values;
+        final Set<Literal> values;
 
         public DiscreteValue(ExpressionRewriteContext context,
-                Expression reference, Expression toExpr, Literal... values) {
-            this(context, reference, toExpr, Arrays.asList(values));
-        }
-
-        public DiscreteValue(ExpressionRewriteContext context,
-                Expression reference, Expression toExpr, Collection<Literal> values) {
-            super(context, reference, toExpr);
-            this.values = Sets.newHashSet(values);
+                Expression reference, Set<Literal> values) {
+            super(context, reference);
+            this.values = values;
         }
 
         public Set<Literal> getValues() {
@@ -372,20 +404,15 @@ public ValueDesc union(ValueDesc other) {
                 return other.union(this);
             }
             if (other instanceof DiscreteValue) {
-                Expression originExpr = FoldConstantRuleOnFE.evaluate(
-                        ExpressionUtils.or(toExpr, other.toExpr), context);
-                DiscreteValue discreteValue = new DiscreteValue(context, reference, originExpr);
-                discreteValue.values.addAll(((DiscreteValue) other).values);
-                discreteValue.values.addAll(this.values);
-                return discreteValue;
+                Set<Literal> newValues = Sets.newHashSet();
+                newValues.addAll(((DiscreteValue) other).values);
+                newValues.addAll(this.values);
+                return new DiscreteValue(context, reference, newValues);
             }
             if (other instanceof RangeValue) {
                 return union(context, (RangeValue) other, this, true);
             }
-            Expression originExpr = FoldConstantRuleOnFE.evaluate(
-                    ExpressionUtils.or(toExpr, other.toExpr), context);
-            return new UnknownValue(context, originExpr,
-                    ImmutableList.of(this, other), false);
+            return new UnknownValue(context, ImmutableList.of(this, other), false);
         }
 
         @Override
@@ -394,24 +421,19 @@ public ValueDesc intersect(ValueDesc other) {
                 return other.intersect(this);
             }
             if (other instanceof DiscreteValue) {
-                Expression originExpr = FoldConstantRuleOnFE.evaluate(
-                        ExpressionUtils.and(toExpr, other.toExpr), context);
-                DiscreteValue discreteValue = new DiscreteValue(context, reference, originExpr);
-                discreteValue.values.addAll(((DiscreteValue) other).values);
-                discreteValue.values.retainAll(this.values);
-                if (discreteValue.values.isEmpty()) {
-                    return new EmptyValue(context, reference, originExpr);
+                Set<Literal> newValues = Sets.newHashSet();
+                newValues.addAll(((DiscreteValue) other).values);
+                newValues.retainAll(this.values);
+                if (newValues.isEmpty()) {
+                    return new EmptyValue(context, reference);
                 } else {
-                    return discreteValue;
+                    return new DiscreteValue(context, reference, newValues);
                 }
             }
             if (other instanceof RangeValue) {
                 return intersect(context, (RangeValue) other, this);
             }
-            Expression originExpr = FoldConstantRuleOnFE.evaluate(
-                    ExpressionUtils.and(toExpr, other.toExpr), context);
-            return new UnknownValue(context, originExpr,
-                    ImmutableList.of(this, other), true);
+            return new UnknownValue(context, ImmutableList.of(this, other), true);
         }
 
         @Override
@@ -428,14 +450,14 @@ public static class UnknownValue extends ValueDesc {
         private final boolean isAnd;
 
         private UnknownValue(ExpressionRewriteContext context, Expression expr) {
-            super(context, expr, expr);
+            super(context, expr);
             sourceValues = ImmutableList.of();
             isAnd = false;
         }
 
-        public UnknownValue(ExpressionRewriteContext context, Expression toExpr,
+        private UnknownValue(ExpressionRewriteContext context,
                 List<ValueDesc> sourceValues, boolean isAnd) {
-            super(context, getReference(sourceValues, toExpr), toExpr);
+            super(context, getReference(context, sourceValues, isAnd));
             this.sourceValues = ImmutableList.copyOf(sourceValues);
             this.isAnd = isAnd;
         }
@@ -455,11 +477,12 @@ public UnknownValue(ExpressionRewriteContext context, Expression toExpr,
         //    E union UnknownValue1 = E.union(UnknownValue1) = UnknownValue1,
         // 2. since E and UnknownValue2's reference not equals, then
         //    E union UnknownValue2 = UnknownValue3(E union UnknownValue2, reference=E union UnknownValue2)
-        private static Expression getReference(List<ValueDesc> sourceValues, Expression toExpr) {
+        private static Expression getReference(ExpressionRewriteContext context,
+                List<ValueDesc> sourceValues, boolean isAnd) {
             Expression reference = sourceValues.get(0).reference;
             for (int i = 1; i < sourceValues.size(); i++) {
                 if (!reference.equals(sourceValues.get(i).reference)) {
-                    return toExpr;
+                    return SimplifyRange.INSTANCE.getExpression(context, sourceValues, isAnd);
                 }
             }
             return reference;
@@ -480,10 +503,7 @@ public ValueDesc union(ValueDesc other) {
             if (other instanceof EmptyValue) {
                 return other.union(this);
             }
-            Expression originExpr = FoldConstantRuleOnFE.evaluate(
-                    ExpressionUtils.or(toExpr, other.toExpr), context);
-            return new UnknownValue(context, originExpr,
-                    ImmutableList.of(this, other), false);
+            return new UnknownValue(context, ImmutableList.of(this, other), false);
         }
 
         @Override
@@ -493,10 +513,7 @@ public ValueDesc intersect(ValueDesc other) {
             if (other instanceof EmptyValue) {
                 return other.intersect(this);
             }
-            Expression originExpr = FoldConstantRuleOnFE.evaluate(
-                    ExpressionUtils.and(toExpr, other.toExpr), context);
-            return new UnknownValue(context, originExpr,
-                    ImmutableList.of(this, other), true);
+            return new UnknownValue(context, ImmutableList.of(this, other), true);
         }
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java
index 576ef6bbf4d5df8..64891882f7d6611 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java
@@ -38,6 +38,7 @@
 import org.apache.doris.nereids.trees.expressions.literal.Literal;
 import org.apache.doris.nereids.util.ExpressionUtils;
 
+import com.google.common.base.Preconditions;
 import com.google.common.collect.BoundType;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
@@ -150,26 +151,28 @@ private Expression getExpression(DiscreteValue value) {
 
     private Expression getExpression(UnknownValue value) {
         List<ValueDesc> sourceValues = value.getSourceValues();
-        Expression originExpr = value.getOriginExpr();
         if (sourceValues.isEmpty()) {
-            return originExpr;
+            return value.getReference();
+        } else {
+            return getExpression(value.getExpressionRewriteContext(), sourceValues, value.isAnd());
         }
+    }
+
+    /** getExpression */
+    public Expression getExpression(ExpressionRewriteContext context,
+            List<ValueDesc> sourceValues, boolean isAnd) {
+        Preconditions.checkArgument(!sourceValues.isEmpty());
         List<Expression> sourceExprs = Lists.newArrayListWithExpectedSize(sourceValues.size());
         for (ValueDesc sourceValue : sourceValues) {
             Expression expr = getExpression(sourceValue);
-            if (value.isAnd()) {
+            if (isAnd) {
                 sourceExprs.addAll(ExpressionUtils.extractConjunction(expr));
             } else {
                 sourceExprs.addAll(ExpressionUtils.extractDisjunction(expr));
             }
         }
-        Expression result = value.isAnd() ? ExpressionUtils.and(sourceExprs) : ExpressionUtils.or(sourceExprs);
-        result = FoldConstantRuleOnFE.evaluate(result, value.getExpressionRewriteContext());
-        // ATTN: we must return original expr, because OrToIn is implemented with MutableState,
-        //   newExpr will lose these states leading to dead loop by OrToIn -> SimplifyRange -> FoldConstantByFE
-        if (result.equals(originExpr)) {
-            return originExpr;
-        }
+        Expression result = isAnd ? ExpressionUtils.and(sourceExprs) : ExpressionUtils.or(sourceExprs);
+        result = FoldConstantRuleOnFE.evaluate(result, context);
         return result;
     }
 }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java
index 784600577c37c62..7393439c5e617f6 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java
@@ -22,6 +22,10 @@
 import org.apache.doris.nereids.analyzer.UnboundSlot;
 import org.apache.doris.nereids.parser.NereidsParser;
 import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer;
+import org.apache.doris.nereids.rules.expression.rules.RangeInference;
+import org.apache.doris.nereids.rules.expression.rules.RangeInference.EmptyValue;
+import org.apache.doris.nereids.rules.expression.rules.RangeInference.UnknownValue;
+import org.apache.doris.nereids.rules.expression.rules.RangeInference.ValueDesc;
 import org.apache.doris.nereids.rules.expression.rules.SimplifyRange;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.InPredicate;
@@ -60,6 +64,24 @@ public SimplifyRangeTest() {
         context = new ExpressionRewriteContext(cascadesContext);
     }
 
+    @Test
+    public void testRangeInference() {
+        ValueDesc valueDesc = getValueDesc("TA IS NULL");
+        Assertions.assertInstanceOf(UnknownValue.class, valueDesc);
+        List<ValueDesc> sourceValues = ((UnknownValue) valueDesc).getSourceValues();
+        Assertions.assertEquals(0, sourceValues.size());
+        Assertions.assertEquals("TA IS NULL", valueDesc.getReference().toSql());
+
+        valueDesc = getValueDesc("TA IS NULL AND TB IS NULL AND NULL");
+        Assertions.assertInstanceOf(UnknownValue.class, valueDesc);
+        sourceValues = ((UnknownValue) valueDesc).getSourceValues();
+        Assertions.assertEquals(3, sourceValues.size());
+        Assertions.assertInstanceOf(EmptyValue.class, sourceValues.get(0));
+        Assertions.assertInstanceOf(EmptyValue.class, sourceValues.get(1));
+        Assertions.assertEquals("TA", sourceValues.get(0).getReference().toSql());
+        Assertions.assertEquals("TB", sourceValues.get(1).getReference().toSql());
+    }
+
     @Test
     public void testSimplify() {
         executor = new ExpressionRuleExecutor(ImmutableList.of(
@@ -69,8 +91,15 @@ public void testSimplify() {
         assertRewrite("TA > 3 or TA > null", "TA > 3 OR NULL");
         assertRewrite("TA > 3 or TA < null", "TA > 3 OR NULL");
         assertRewrite("TA > 3 or TA = null", "TA > 3 OR NULL");
+        assertRewrite("TA > 3 or TA = 3 or TA < null", "TA >= 3 OR NULL");
+        assertRewrite("TA < 10 or TA in (1, 2, 3, 11, 12, 13)", "TA in (11, 12, 13) OR TA < 10");
+        assertRewrite("TA < 10 or TA in (1, 2, 3, 10, 11, 12, 13) or TA > 13 or TA < 10 or TA in (1, 2, 3, 10, 11, 12, 13) or TA > 13",
+                "TA in (11, 12) OR TA <= 10 OR TA >= 13");
         assertRewrite("TA > 3 or TA <> null", "TA > 3 or null");
         assertRewrite("TA > 3 or TA <=> null", "TA > 3 or TA <=> null");
+        assertRewrite("(TA < 1 or TA > 2) or (TA >= 0 and TA <= 3)", "TA IS NOT NULL OR NULL");
+        assertRewrite("TA between 10 and 20 or TA between 100 and 120 or TA between 15 and 25 or TA between 115 and 125",
+                "TA between 10 and 25 or TA between 100 and 125");
         assertRewriteNotNull("TA > 3 and TA > null", "TA > 3 and NULL");
         assertRewriteNotNull("TA > 3 and TA < null", "TA > 3 and NULL");
         assertRewriteNotNull("TA > 3 and TA = null", "TA > 3 and NULL");
@@ -88,13 +117,13 @@ public void testSimplify() {
         assertRewrite("TA >= 3 and TA < 3", "TA >= 3 and TA < 3");
         assertRewriteNotNull("TA = 1 and TA > 10", "FALSE");
         assertRewrite("TA = 1 and TA > 10", "TA is null and null");
-        assertRewrite("TA > 5 or TA < 1", "TA > 5 or TA < 1");
+        assertRewrite("TA > 5 or TA < 1", "TA < 1 or TA > 5");
         assertRewrite("TA > 5 or TA > 1 or TA > 10", "TA > 1");
         assertRewrite("TA > 5 or TA > 1 or TA < 10", "TA is not null or null");
         assertRewriteNotNull("TA > 5 or TA > 1 or TA < 10", "TRUE");
         assertRewrite("TA > 5 and TA > 1 and TA > 10", "TA > 10");
         assertRewrite("TA > 5 and TA > 1 and TA < 10", "TA > 5 and TA < 10");
-        assertRewrite("TA > 1 or TA < 1", "TA > 1 or TA < 1");
+        assertRewrite("TA > 1 or TA < 1", "TA < 1 or TA > 1");
         assertRewrite("TA > 1 or TA < 10", "TA is not null or null");
         assertRewriteNotNull("TA > 1 or TA < 10", "TRUE");
         assertRewrite("TA > 5 and TA < 10", "TA > 5 and TA < 10");
@@ -109,7 +138,7 @@ public void testSimplify() {
         assertRewrite("(TA > 10 or TA > 20) and (TB > 10 and TB > 20)", "TA > 10 and TB > 20");
         assertRewrite("((TB > 30 and TA > 40) and TA > 20) and (TB > 10 and TB > 20)", "TB > 30 and TA > 40");
         assertRewrite("(TA > 10 and TB > 10) or (TB > 10 and TB > 20)", "TA > 10 and TB > 10 or TB > 20");
-        assertRewrite("((TA > 10 or TA > 5) and TB > 10) or (TB > 10 and (TB > 20 or TB < 10))", "(TA > 5 and TB > 10) or (TB > 10 and (TB > 20 or TB < 10))");
+        assertRewrite("((TA > 10 or TA > 5) and TB > 10) or (TB > 10 and (TB > 20 or TB < 10))", "(TA > 5 and TB > 10) or (TB > 10 and (TB < 10 or TB > 20))");
         assertRewriteNotNull("TA in (1,2,3) and TA > 10", "FALSE");
         assertRewrite("TA in (1,2,3) and TA > 10", "TA is null and null");
         assertRewrite("TA in (1,2,3) and TA >= 1", "TA in (1,2,3)");
@@ -119,7 +148,7 @@ public void testSimplify() {
         assertRewrite("TA in (1,2,3) and TA < 10", "TA in (1,2,3)");
         assertRewriteNotNull("TA in (1,2,3) and TA < 1", "FALSE");
         assertRewrite("TA in (1,2,3) and TA < 1", "TA is null and null");
-        assertRewrite("TA in (1,2,3) or TA < 1", "TA in (1,2,3) or TA < 1");
+        assertRewrite("TA in (1,2,3) or TA < 1", "TA in (2,3) or TA <= 1");
         assertRewrite("TA in (1,2,3) or TA in (2,3,4)", "TA in (1,2,3,4)");
         assertRewrite("TA in (1,2,3) or TA in (4,5,6)", "TA in (1,2,3,4,5,6)");
         assertRewrite("TA in (1,2,3) and TA in (4,5,6)", "TA is null and null");
@@ -150,12 +179,12 @@ public void testSimplify() {
         assertRewrite("TA + TC >= 3 and TA + TC < 3", "TA + TC >= 3 and TA + TC < 3");
         assertRewriteNotNull("TA + TC = 1 and TA + TC > 10", "FALSE");
         assertRewrite("TA + TC = 1 and TA + TC > 10", "(TA + TC) is null and null");
-        assertRewrite("TA + TC > 5 or TA + TC < 1", "TA + TC > 5 or TA + TC < 1");
+        assertRewrite("TA + TC > 5 or TA + TC < 1", "TA + TC < 1 or TA + TC > 5");
         assertRewrite("TA + TC > 5 or TA + TC > 1 or TA + TC > 10", "TA + TC > 1");
         assertRewrite("TA + TC > 5 or TA + TC > 1 or TA + TC < 10", "(TA + TC) is not null or null");
         assertRewrite("TA + TC > 5 and TA + TC > 1 and TA + TC > 10", "TA + TC > 10");
         assertRewrite("TA + TC > 5 and TA + TC > 1 and TA + TC < 10", "TA + TC > 5 and TA + TC < 10");
-        assertRewrite("TA + TC > 1 or TA + TC < 1", "TA + TC > 1 or TA + TC < 1");
+        assertRewrite("TA + TC > 1 or TA + TC < 1", "TA + TC < 1 or TA + TC > 1");
         assertRewrite("TA + TC > 1 or TA + TC < 10", "(TA + TC) is not null or null");
         assertRewrite("TA + TC > 5 and TA + TC < 10", "TA + TC > 5 and TA + TC < 10");
         assertRewrite("TA + TC > 5 and TA + TC > 10", "TA + TC > 10");
@@ -168,7 +197,7 @@ public void testSimplify() {
         assertRewrite("(TA + TC > 10 or TA + TC > 20) and (TB > 10 and TB > 20)", "TA + TC > 10 and TB > 20");
         assertRewrite("((TB > 30 and TA + TC > 40) and TA + TC > 20) and (TB > 10 and TB > 20)", "TB > 30 and TA + TC > 40");
         assertRewrite("(TA + TC > 10 and TB > 10) or (TB > 10 and TB > 20)", "TA + TC > 10 and TB > 10 or TB > 20");
-        assertRewrite("((TA + TC > 10 or TA + TC > 5) and TB > 10) or (TB > 10 and (TB > 20 or TB < 10))", "(TA + TC > 5 and TB > 10) or (TB > 10 and (TB > 20 or TB < 10))");
+        assertRewrite("((TA + TC > 10 or TA + TC > 5) and TB > 10) or (TB > 10 and (TB > 20 or TB < 10))", "(TA + TC > 5 and TB > 10) or (TB > 10 and (TB < 10 or TB > 20))");
         assertRewriteNotNull("TA + TC in (1,2,3) and TA + TC > 10", "FALSE");
         assertRewrite("TA + TC in (1,2,3) and TA + TC > 10", "(TA + TC) is null and null");
         assertRewrite("TA + TC in (1,2,3) and TA + TC >= 1", "TA + TC in (1,2,3)");
@@ -178,7 +207,7 @@ public void testSimplify() {
         assertRewrite("TA + TC in (1,2,3) and TA + TC < 10", "TA + TC in (1,2,3)");
         assertRewriteNotNull("TA + TC in (1,2,3) and TA + TC < 1", "FALSE");
         assertRewrite("TA + TC in (1,2,3) and TA + TC < 1", "(TA + TC) is null and null");
-        assertRewrite("TA + TC in (1,2,3) or TA + TC < 1", "TA + TC in (1,2,3) or TA + TC < 1");
+        assertRewrite("TA + TC in (1,2,3) or TA + TC < 1", "TA + TC in (2,3) or TA + TC <= 1");
         assertRewrite("TA + TC in (1,2,3) or TA + TC in (2,3,4)", "TA + TC in (1,2,3,4)");
         assertRewrite("TA + TC in (1,2,3) or TA + TC in (4,5,6)", "TA + TC in (1,2,3,4,5,6)");
         assertRewriteNotNull("TA + TC in (1,2,3) and TA + TC in (4,5,6)", "FALSE");
@@ -221,7 +250,7 @@ public void testSimplifyDate() {
         assertRewriteNotNull("AA = date '2024-01-01' and AA > date '2024-01-10'", "FALSE");
         assertRewrite("AA = date '2024-01-01' and AA > date '2024-01-10'", "AA is null and null");
         assertRewrite("AA > date '2024-01-05' or AA < date '2024-01-01'",
-                "AA > date '2024-01-05' or AA < date '2024-01-01'");
+                "AA < date '2024-01-01' or AA > date '2024-01-05'");
         assertRewrite("AA > date '2024-01-05' or AA > date '2024-01-01' or AA > date '2024-01-10'",
                 "AA > date '2024-01-01'");
         assertRewrite("AA > date '2024-01-05' or AA > date '2024-01-01' or AA < date '2024-01-10'", "AA is not null or null");
@@ -231,7 +260,7 @@ public void testSimplifyDate() {
         assertRewrite("AA > date '2024-01-05' and AA > date '2024-01-01' and AA < date '2024-01-10'",
                 "AA > date '2024-01-05' and AA < date '2024-01-10'");
         assertRewrite("AA > date '2024-01-05' or AA < date '2024-01-05'",
-                "AA > date '2024-01-05' or AA < date '2024-01-05'");
+                "AA < date '2024-01-05' or AA > date '2024-01-05'");
         assertRewrite("AA > date '2024-01-01' or AA < date '2024-01-10'", "AA is not null or null");
         assertRewriteNotNull("AA > date '2024-01-01' or AA < date '2024-01-10'", "TRUE");
         assertRewrite("AA > date '2024-01-05' and AA < date '2024-01-10'",
@@ -261,7 +290,7 @@ public void testSimplifyDate() {
         assertRewrite("AA in (date '2024-01-01',date '2024-01-02',date '2024-01-03') and AA < date '2024-01-01'",
                 "AA is null and null");
         assertRewrite("AA in (date '2024-01-01',date '2024-01-02',date '2024-01-03') or AA < date '2024-01-01'",
-                "AA in (date '2024-01-01',date '2024-01-02',date '2024-01-03') or AA < date '2024-01-01'");
+                "AA in (date '2024-01-02',date '2024-01-03') or AA <= date '2024-01-01'");
         assertRewrite("AA in (date '2024-01-01',date '2024-01-02') or AA in (date '2024-01-02', date '2024-01-03')",
                 "AA in (date '2024-01-01',date '2024-01-02',date '2024-01-03')");
         assertRewriteNotNull("AA in (date '2024-01-01',date '2024-01-02') and AA in (date '2024-01-03', date '2024-01-04')",
@@ -301,7 +330,7 @@ public void testSimplifyDateTime() {
         assertRewriteNotNull("CA = timestamp '2024-01-01 10:00:10' and CA > timestamp '2024-01-10 00:00:10'", "FALSE");
         assertRewrite("CA = timestamp '2024-01-01 10:00:10' and CA > timestamp '2024-01-10 00:00:10'", "CA is null and null");
         assertRewrite("CA > timestamp '2024-01-05 00:00:10' or CA < timestamp '2024-01-01 00:00:10'",
-                "CA > timestamp '2024-01-05 00:00:10' or CA < timestamp '2024-01-01 00:00:10'");
+                "CA < timestamp '2024-01-01 00:00:10' or CA > timestamp '2024-01-05 00:00:10'");
         assertRewrite("CA > timestamp '2024-01-05 00:00:10' or CA > timestamp '2024-01-01 00:00:10' or CA > timestamp '2024-01-10 00:00:10'",
                 "CA > timestamp '2024-01-01 00:00:10'");
         assertRewrite("CA > timestamp '2024-01-05 00:00:10' or CA > timestamp '2024-01-01 00:00:10' or CA < timestamp '2024-01-10 00:00:10'", "CA is not null or null");
@@ -311,7 +340,7 @@ public void testSimplifyDateTime() {
         assertRewrite("CA > timestamp '2024-01-05 00:00:10' and CA > timestamp '2024-01-01 00:00:10' and CA < timestamp '2024-01-10 00:00:10'",
                 "CA > timestamp '2024-01-05 00:00:10' and CA < timestamp '2024-01-10 00:00:10'");
         assertRewrite("CA > timestamp '2024-01-05 00:00:10' or CA < timestamp '2024-01-05 00:00:10'",
-                "CA > timestamp '2024-01-05 00:00:10' or CA < timestamp '2024-01-05 00:00:10'");
+                "CA < timestamp '2024-01-05 00:00:10' or CA > timestamp '2024-01-05 00:00:10'");
         assertRewrite("CA > timestamp '2024-01-01 00:02:10' or CA < timestamp '2024-01-10 00:02:10'", "CA is not null or null");
         assertRewriteNotNull("CA > timestamp '2024-01-01 00:00:00' or CA < timestamp '2024-01-10 00:00:00'", "TRUE");
         assertRewrite("CA > timestamp '2024-01-05 01:00:00' and CA < timestamp '2024-01-10 01:00:00'",
@@ -364,7 +393,13 @@ public void testSimplifyDateTime() {
                 "(CA is null and null) OR CB < timestamp '2024-01-05 00:50:00'");
     }
 
-    @Test
+    private ValueDesc getValueDesc(String expression) {
+        Map<String, Slot> mem = Maps.newHashMap();
+        Expression parseExpression = replaceUnboundSlot(PARSER.parseExpression(expression), mem);
+        parseExpression = typeCoercion(parseExpression);
+        return (new RangeInference()).getValue(parseExpression, context);
+    }
+
     private void assertRewrite(String expression, String expected) {
         Map<String, Slot> mem = Maps.newHashMap();
         Expression needRewriteExpression = replaceUnboundSlot(PARSER.parseExpression(expression), mem);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java
index 3d88c131c97f153..cdc36164ae9f95e 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java
@@ -58,7 +58,7 @@ void testInferNotNullFromFilterAndEliminateOuter2() {
                                 f -> ExpressionUtils.and(f.getConjuncts().stream()
                                         .sorted((a, b) -> a.toString().compareTo(b.toString()))
                                         .collect(Collectors.toList()))
-                                        .toString().equals("AND[(id#0 >= 4),OR[(id#0 = 4),(id#0 > 4)]]"))
+                                        .toString().equals("(id#0 >= 4)"))
                     )
 
                 );
@@ -76,7 +76,7 @@ void testInferNotNullFromFilterAndEliminateOuter3() {
                         logicalFilter(
                             leftOuterLogicalJoin(
                                 logicalFilter().when(
-                                        f -> f.getPredicate().toString().equals("AND[(id#0 >= 4),OR[(id#0 = 4),(id#0 > 4)]]")),
+                                        f -> f.getPredicate().toString().equals("(id#0 >= 4)")),
                                 logicalFilter().when(
                                         f -> f.getPredicate().toString().equals("(id#2 >= 4)")
                                 )

From 75ef6bd0b88ade3bf6e306681713c82aefce1aa0 Mon Sep 17 00:00:00 2001
From: meiyi <meiyi@selectdb.com>
Date: Thu, 9 Jan 2025 12:10:38 +0800
Subject: [PATCH 066/140] [fix](group commit) support full prepare of group
 commit in nereids (#46533)

---
 .../doris/analysis/NativeInsertStmt.java      |   7 +-
 .../doris/cloud/catalog/CloudEnvFactory.java  |  13 --
 .../planner/CloudGroupCommitPlanner.java      |  40 ----
 .../trees/plans/commands/ExecuteCommand.java  |  48 ++--
 .../insert/InsertIntoTableCommand.java        |  31 +++
 .../insert/OlapGroupCommitInsertExecutor.java |  42 +++-
 .../doris/planner/GroupCommitPlanner.java     | 207 +++++++++++++++---
 .../doris/qe/PreparedStatementContext.java    |   2 +
 .../org/apache/doris/qe/SessionVariable.java  |   4 +
 .../org/apache/doris/qe/StmtExecutor.java     |  10 +-
 .../insert_group_commit_with_exception.groovy |   1 -
 ...sert_group_commit_with_prepare_stmt.groovy |   4 +-
 .../transaction/txn_insert_inject_case.groovy |   2 +-
 13 files changed, 283 insertions(+), 128 deletions(-)
 delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/cloud/planner/CloudGroupCommitPlanner.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java
index 14680f54b1dac6e..20207e672634c33 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java
@@ -154,7 +154,6 @@ public class NativeInsertStmt extends InsertStmt {
 
     // Used for group commit insert
     private boolean isGroupCommit = false;
-    private int baseSchemaVersion = -1;
     private TUniqueId loadId = null;
     private long tableId = -1;
     public boolean isGroupCommitStreamLoadSql = false;
@@ -1304,7 +1303,8 @@ public GroupCommitPlanner planForGroupCommit(TUniqueId queryId) throws UserExcep
         OlapTable olapTable = (OlapTable) getTargetTable();
         olapTable.readLock();
         try {
-            if (groupCommitPlanner != null && olapTable.getBaseSchemaVersion() == baseSchemaVersion) {
+            if (groupCommitPlanner != null
+                    && olapTable.getBaseSchemaVersion() == groupCommitPlanner.baseSchemaVersion) {
                 if (LOG.isDebugEnabled()) {
                     LOG.debug("reuse group commit plan, table={}", olapTable);
                 }
@@ -1322,7 +1322,6 @@ public GroupCommitPlanner planForGroupCommit(TUniqueId queryId) throws UserExcep
                     targetColumnNames, queryId, ConnectContext.get().getSessionVariable().getGroupCommit());
             // save plan message to be reused for prepare stmt
             loadId = queryId;
-            baseSchemaVersion = olapTable.getBaseSchemaVersion();
             return groupCommitPlanner;
         } finally {
             olapTable.readUnlock();
@@ -1334,7 +1333,7 @@ public TUniqueId getLoadId() {
     }
 
     public int getBaseSchemaVersion() {
-        return baseSchemaVersion;
+        return groupCommitPlanner.baseSchemaVersion;
     }
 
     public void setIsFromDeleteOrUpdateStmt(boolean isFromDeleteOrUpdateStmt) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudEnvFactory.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudEnvFactory.java
index 32992307a8b203a..764ea49f329a358 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudEnvFactory.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudEnvFactory.java
@@ -22,11 +22,9 @@
 import org.apache.doris.analysis.DescriptorTable;
 import org.apache.doris.analysis.UserIdentity;
 import org.apache.doris.catalog.CloudTabletStatMgr;
-import org.apache.doris.catalog.Database;
 import org.apache.doris.catalog.DynamicPartitionProperty;
 import org.apache.doris.catalog.Env;
 import org.apache.doris.catalog.EnvFactory;
-import org.apache.doris.catalog.OlapTable;
 import org.apache.doris.catalog.Partition;
 import org.apache.doris.catalog.Replica;
 import org.apache.doris.catalog.ReplicaAllocation;
@@ -37,12 +35,10 @@
 import org.apache.doris.cloud.load.CloudBrokerLoadJob;
 import org.apache.doris.cloud.load.CloudLoadManager;
 import org.apache.doris.cloud.load.CloudRoutineLoadManager;
-import org.apache.doris.cloud.planner.CloudGroupCommitPlanner;
 import org.apache.doris.cloud.qe.CloudCoordinator;
 import org.apache.doris.cloud.system.CloudSystemInfoService;
 import org.apache.doris.cloud.transaction.CloudGlobalTransactionMgr;
 import org.apache.doris.common.MetaNotFoundException;
-import org.apache.doris.common.UserException;
 import org.apache.doris.common.util.MasterDaemon;
 import org.apache.doris.common.util.PropertyAnalyzer;
 import org.apache.doris.datasource.InternalCatalog;
@@ -51,7 +47,6 @@
 import org.apache.doris.load.loadv2.LoadManager;
 import org.apache.doris.load.routineload.RoutineLoadManager;
 import org.apache.doris.nereids.stats.StatsErrorEstimator;
-import org.apache.doris.planner.GroupCommitPlanner;
 import org.apache.doris.planner.PlanFragment;
 import org.apache.doris.planner.Planner;
 import org.apache.doris.planner.ScanNode;
@@ -62,8 +57,6 @@
 import org.apache.doris.thrift.TUniqueId;
 import org.apache.doris.transaction.GlobalTransactionMgrIface;
 
-import org.apache.thrift.TException;
-
 import java.lang.reflect.Type;
 import java.util.List;
 import java.util.Map;
@@ -168,12 +161,6 @@ public Coordinator createCoordinator(Long jobId, TUniqueId queryId, DescriptorTa
                                 enableProfile);
     }
 
-    @Override
-    public GroupCommitPlanner createGroupCommitPlanner(Database db, OlapTable table, List<String> targetColumnNames,
-            TUniqueId queryId, String groupCommit) throws UserException, TException {
-        return  new CloudGroupCommitPlanner(db, table, targetColumnNames, queryId, groupCommit);
-    }
-
     @Override
     public RoutineLoadManager createRoutineLoadManager() {
         return new CloudRoutineLoadManager();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/planner/CloudGroupCommitPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/planner/CloudGroupCommitPlanner.java
deleted file mode 100644
index 0388ca5c5d68099..000000000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/planner/CloudGroupCommitPlanner.java
+++ /dev/null
@@ -1,40 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.cloud.planner;
-
-import org.apache.doris.catalog.Database;
-import org.apache.doris.catalog.OlapTable;
-import org.apache.doris.common.UserException;
-import org.apache.doris.planner.GroupCommitPlanner;
-import org.apache.doris.thrift.TUniqueId;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.apache.thrift.TException;
-
-import java.util.List;
-
-public class CloudGroupCommitPlanner extends GroupCommitPlanner {
-    private static final Logger LOG = LogManager.getLogger(CloudGroupCommitPlanner.class);
-
-    public CloudGroupCommitPlanner(Database db, OlapTable table, List<String> targetColumnNames, TUniqueId queryId,
-            String groupCommit)
-            throws UserException, TException {
-        super(db, table, targetColumnNames, queryId, groupCommit);
-    }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java
index 47ba6ed3f4c2ac3..c4031c0f9e59b81 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java
@@ -24,7 +24,9 @@
 import org.apache.doris.nereids.glue.LogicalPlanAdapter;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.commands.insert.OlapGroupCommitInsertExecutor;
 import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.planner.GroupCommitPlanner;
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.qe.PointQueryExecutor;
 import org.apache.doris.qe.PreparedStatementContext;
@@ -72,27 +74,35 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
         executor.setParsedStmt(planAdapter);
         // If it's not a short circuit query or schema version is different(indicates schema changed) or
         // has nondeterministic functions in statement, then need to do reanalyze and plan
-        boolean isShortCircuit = executor.getContext().getStatementContext().isShortCircuitQuery();
-        boolean hasShortCircuitContext = preparedStmtCtx.shortCircuitQueryContext.isPresent();
-        boolean schemaVersionMismatch = hasShortCircuitContext
-                    && preparedStmtCtx.shortCircuitQueryContext.get().tbl.getBaseSchemaVersion()
-                    != preparedStmtCtx.shortCircuitQueryContext.get().schemaVersion;
-        boolean needAnalyze = !isShortCircuit || schemaVersionMismatch || !hasShortCircuitContext
-                        || executor.getContext().getStatementContext().hasNondeterministic();
-        if (needAnalyze) {
-            // execute real statement
-            preparedStmtCtx.shortCircuitQueryContext = Optional.empty();
-            statementContext.setShortCircuitQueryContext(null);
-            executor.execute();
-            if (executor.getContext().getStatementContext().isShortCircuitQuery()) {
-                // cache short-circuit plan
-                preparedStmtCtx.shortCircuitQueryContext = Optional.of(
-                        new ShortCircuitQueryContext(executor.planner(), (Queriable) executor.getParsedStmt()));
-                statementContext.setShortCircuitQueryContext(preparedStmtCtx.shortCircuitQueryContext.get());
-            }
+        if (executor.getContext().getStatementContext().isShortCircuitQuery()
+                && preparedStmtCtx.shortCircuitQueryContext.isPresent()
+                && preparedStmtCtx.shortCircuitQueryContext.get().tbl.getBaseSchemaVersion()
+                == preparedStmtCtx.shortCircuitQueryContext.get().schemaVersion && !executor.getContext()
+                .getStatementContext().hasNondeterministic()) {
+            PointQueryExecutor.directExecuteShortCircuitQuery(executor, preparedStmtCtx, statementContext);
             return;
         }
-        PointQueryExecutor.directExecuteShortCircuitQuery(executor, preparedStmtCtx, statementContext);
+        if (ctx.getSessionVariable().enableGroupCommitFullPrepare) {
+            if (preparedStmtCtx.groupCommitPlanner.isPresent()) {
+                OlapGroupCommitInsertExecutor.fastAnalyzeGroupCommit(ctx, prepareCommand);
+            } else {
+                OlapGroupCommitInsertExecutor.analyzeGroupCommit(ctx, prepareCommand);
+            }
+            if (ctx.isGroupCommit()) {
+                GroupCommitPlanner.executeGroupCommitInsert(ctx, preparedStmtCtx, statementContext);
+                return;
+            }
+        }
+        // execute real statement
+        preparedStmtCtx.shortCircuitQueryContext = Optional.empty();
+        statementContext.setShortCircuitQueryContext(null);
+        executor.execute();
+        if (executor.getContext().getStatementContext().isShortCircuitQuery()) {
+            // cache short-circuit plan
+            preparedStmtCtx.shortCircuitQueryContext = Optional.of(
+                    new ShortCircuitQueryContext(executor.planner(), (Queriable) executor.getParsedStmt()));
+            statementContext.setShortCircuitQueryContext(preparedStmtCtx.shortCircuitQueryContext.get());
+        }
     }
 
     /**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java
index c5c13d1a4a1577d..c65ce4a282ddf86 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java
@@ -47,6 +47,7 @@
 import org.apache.doris.nereids.trees.plans.commands.Command;
 import org.apache.doris.nereids.trees.plans.commands.ForwardWithSync;
 import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.trees.plans.logical.UnboundLogicalSink;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalHiveTableSink;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalIcebergTableSink;
@@ -67,6 +68,7 @@
 import com.google.common.base.Preconditions;
 import com.google.common.base.Throwables;
 import com.google.common.collect.Lists;
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -423,6 +425,35 @@ public boolean isExternalTableSink() {
         return !(getLogicalQuery() instanceof UnboundTableSink);
     }
 
+    /**
+     * get the target table of the insert command
+     */
+    public TableIf getTable(ConnectContext ctx) throws Exception {
+        TableIf targetTableIf = InsertUtils.getTargetTable(originLogicalQuery, ctx);
+        if (!Env.getCurrentEnv().getAccessManager()
+                .checkTblPriv(ConnectContext.get(), targetTableIf.getDatabase().getCatalog().getName(),
+                        targetTableIf.getDatabase().getFullName(), targetTableIf.getName(),
+                        PrivPredicate.LOAD)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "LOAD",
+                    ConnectContext.get().getQualifiedUser(), ConnectContext.get().getRemoteIP(),
+                    targetTableIf.getDatabase().getFullName() + "." + targetTableIf.getName());
+        }
+        return targetTableIf;
+    }
+
+    /**
+     * get the target columns of the insert command
+     */
+    public List<String> getTargetColumns() {
+        if (originLogicalQuery instanceof UnboundTableSink) {
+            UnboundLogicalSink<? extends Plan> unboundTableSink = (UnboundTableSink<? extends Plan>) originLogicalQuery;
+            return CollectionUtils.isEmpty(unboundTableSink.getColNames()) ? null : unboundTableSink.getColNames();
+        } else {
+            throw new AnalysisException(
+                    "the root of plan should be [UnboundTableSink], but it is " + originLogicalQuery.getType());
+        }
+    }
+
     @Override
     public Plan getExplainPlan(ConnectContext ctx) {
         Optional<CascadesContext> analyzeContext = Optional.of(
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapGroupCommitInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapGroupCommitInsertExecutor.java
index 4cf4dfe536c212d..050fc79132899d8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapGroupCommitInsertExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapGroupCommitInsertExecutor.java
@@ -32,6 +32,7 @@
 import org.apache.doris.nereids.exceptions.AnalysisException;
 import org.apache.doris.nereids.trees.plans.algebra.InlineTable;
 import org.apache.doris.nereids.trees.plans.algebra.OneRowRelation;
+import org.apache.doris.nereids.trees.plans.commands.PrepareCommand;
 import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
 import org.apache.doris.planner.GroupCommitPlanner;
@@ -68,15 +69,40 @@ public OlapGroupCommitInsertExecutor(ConnectContext ctx, Table table,
 
     /**
      * check if the sql can run in group commit mode
-     * @param logicalPlan plan of sql
      */
-    public static void analyzeGroupCommit(LogicalPlan logicalPlan) {
-        ConnectContext ctx = ConnectContext.get();
-        if (ctx.getSessionVariable().isEnableInsertGroupCommit() && logicalPlan instanceof InsertIntoTableCommand) {
-            LogicalPlan logicalQuery = ((InsertIntoTableCommand) logicalPlan).getLogicalQuery();
-            TableIf targetTableIf = InsertUtils.getTargetTable(logicalQuery, ctx);
-            OlapGroupCommitInsertExecutor.analyzeGroupCommit(ctx, targetTableIf, logicalQuery,
-                    Optional.empty());
+    public static void fastAnalyzeGroupCommit(ConnectContext ctx, LogicalPlan logicalPlan) {
+        try {
+            if (ctx.getSessionVariable().isEnableInsertGroupCommit() && !ctx.isTxnModel() && !ctx.getSessionVariable()
+                    .isEnableUniqueKeyPartialUpdate()) {
+                ctx.setGroupCommit(true);
+            }
+        } catch (Throwable e) {
+            LOG.warn("analyze group commit failed", e);
+        }
+    }
+
+    /**
+     * check if the sql can run in group commit mode
+     */
+    public static void analyzeGroupCommit(ConnectContext ctx, LogicalPlan logicalPlan) {
+        try {
+            if (ctx.isGroupCommit()) {
+                return;
+            }
+            if (!ctx.getSessionVariable().isEnableInsertGroupCommit()) {
+                return;
+            }
+            if (logicalPlan instanceof PrepareCommand) {
+                logicalPlan = ((PrepareCommand) logicalPlan).getLogicalPlan();
+            }
+            if (logicalPlan instanceof InsertIntoTableCommand) {
+                LogicalPlan logicalQuery = ((InsertIntoTableCommand) logicalPlan).getLogicalQuery();
+                TableIf targetTableIf = InsertUtils.getTargetTable(logicalQuery, ctx);
+                OlapGroupCommitInsertExecutor.analyzeGroupCommit(ctx, targetTableIf, logicalQuery,
+                        Optional.empty());
+            }
+        } catch (Throwable e) {
+            LOG.warn("analyze group commit failed", e);
         }
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/GroupCommitPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/GroupCommitPlanner.java
index 234f8e99a882e87..82f7864fd3c5646 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/GroupCommitPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/GroupCommitPlanner.java
@@ -24,16 +24,27 @@
 import org.apache.doris.catalog.Column;
 import org.apache.doris.catalog.Database;
 import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.EnvFactory;
 import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.TableIf.TableType;
 import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
 import org.apache.doris.common.FormatOptions;
 import org.apache.doris.common.LoadException;
+import org.apache.doris.common.Pair;
 import org.apache.doris.common.UserException;
+import org.apache.doris.common.util.DebugUtil;
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.nereids.trees.plans.commands.PrepareCommand;
+import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand;
 import org.apache.doris.proto.InternalService;
 import org.apache.doris.proto.InternalService.PGroupCommitInsertRequest;
 import org.apache.doris.proto.InternalService.PGroupCommitInsertResponse;
 import org.apache.doris.proto.Types;
 import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.PreparedStatementContext;
 import org.apache.doris.qe.StmtExecutor;
 import org.apache.doris.rpc.BackendServiceProxy;
 import org.apache.doris.rpc.RpcException;
@@ -47,11 +58,15 @@
 import org.apache.doris.thrift.TPipelineFragmentParams;
 import org.apache.doris.thrift.TPipelineFragmentParamsList;
 import org.apache.doris.thrift.TScanRangeParams;
+import org.apache.doris.thrift.TStatusCode;
 import org.apache.doris.thrift.TStreamLoadPutRequest;
 import org.apache.doris.thrift.TUniqueId;
+import org.apache.doris.transaction.TransactionStatus;
 
 import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
 import com.google.protobuf.ByteString;
+import com.google.protobuf.ProtocolStringList;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.thrift.TException;
@@ -61,6 +76,7 @@
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.stream.Collectors;
@@ -70,12 +86,14 @@
 public class GroupCommitPlanner {
     private static final Logger LOG = LogManager.getLogger(GroupCommitPlanner.class);
     public static final String SCHEMA_CHANGE = " is blocked on schema change";
+    private static final int MAX_RETRY = 3;
 
-    protected Database db;
-    protected OlapTable table;
-    protected TUniqueId loadId;
-    protected Backend backend;
-    private TPipelineFragmentParamsList paramsList;
+    private Database db;
+    private OlapTable table;
+    public int baseSchemaVersion;
+    private int targetColumnSize;
+    private TUniqueId loadId;
+    private long backendId;
     private ByteString execPlanFragmentParamsBytes;
 
     public GroupCommitPlanner(Database db, OlapTable table, List<String> targetColumnNames, TUniqueId queryId,
@@ -83,6 +101,7 @@ public GroupCommitPlanner(Database db, OlapTable table, List<String> targetColum
             throws UserException, TException {
         this.db = db;
         this.table = table;
+        this.baseSchemaVersion = table.getBaseSchemaVersion();
         if (Env.getCurrentEnv().getGroupCommitManager().isBlock(this.table.getId())) {
             String msg = "insert table " + this.table.getId() + SCHEMA_CHANGE;
             LOG.info(msg);
@@ -123,16 +142,16 @@ public GroupCommitPlanner(Database db, OlapTable table, List<String> targetColum
         Preconditions.checkState(scanRangeParams.size() == 1);
         loadId = queryId;
         // see BackendServiceProxy#execPlanFragmentsAsync
-        paramsList = new TPipelineFragmentParamsList();
+        TPipelineFragmentParamsList paramsList = new TPipelineFragmentParamsList();
         paramsList.addToParamsList(tRequest);
         execPlanFragmentParamsBytes = ByteString.copyFrom(new TSerializer().serialize(paramsList));
     }
 
     public PGroupCommitInsertResponse executeGroupCommitInsert(ConnectContext ctx,
             List<InternalService.PDataRow> rows)
-            throws DdlException, RpcException, ExecutionException, InterruptedException {
-        selectBackends(ctx);
-
+            throws DdlException, RpcException, ExecutionException, InterruptedException, LoadException {
+        Backend backend = Env.getCurrentEnv().getGroupCommitManager().selectBackendForGroupCommit(table.getId(), ctx);
+        backendId = backend.getId();
         PGroupCommitInsertRequest request = PGroupCommitInsertRequest.newBuilder()
                 .setExecPlanFragmentRequest(InternalService.PExecPlanFragmentRequest.newBuilder()
                         .setRequest(execPlanFragmentParamsBytes)
@@ -145,21 +164,8 @@ public PGroupCommitInsertResponse executeGroupCommitInsert(ConnectContext ctx,
         return future.get();
     }
 
-    protected void selectBackends(ConnectContext ctx) throws DdlException {
-        try {
-            backend = Env.getCurrentEnv().getGroupCommitManager()
-                    .selectBackendForGroupCommit(this.table.getId(), ctx);
-        } catch (LoadException e) {
-            throw new DdlException("No suitable backend");
-        }
-    }
-
-    public Backend getBackend() {
-        return backend;
-    }
-
-    public TPipelineFragmentParamsList getParamsList() {
-        return paramsList;
+    public long getBackendId() {
+        return backendId;
     }
 
     public List<InternalService.PDataRow> getRows(NativeInsertStmt stmt) throws UserException {
@@ -167,12 +173,7 @@ public List<InternalService.PDataRow> getRows(NativeInsertStmt stmt) throws User
         SelectStmt selectStmt = (SelectStmt) (stmt.getQueryStmt());
         if (selectStmt.getValueList() != null) {
             for (List<Expr> row : selectStmt.getValueList().getRows()) {
-                InternalService.PDataRow data = StmtExecutor.getRowStringValue(row, FormatOptions.getDefault());
-                if (LOG.isDebugEnabled()) {
-                    LOG.debug("add row: [{}]", data.getColList().stream().map(c -> c.getValue())
-                            .collect(Collectors.joining(",")));
-                }
-                rows.add(data);
+                rows.add(getOneRow(row));
             }
         } else {
             List<Expr> exprList = new ArrayList<>();
@@ -183,13 +184,147 @@ public List<InternalService.PDataRow> getRows(NativeInsertStmt stmt) throws User
                     exprList.add(resultExpr);
                 }
             }
-            InternalService.PDataRow data = StmtExecutor.getRowStringValue(exprList, FormatOptions.getDefault());
-            if (LOG.isDebugEnabled()) {
-                LOG.debug("add row: [{}]", data.getColList().stream().map(c -> c.getValue())
-                        .collect(Collectors.joining(",")));
-            }
-            rows.add(data);
+            rows.add(getOneRow(exprList));
         }
         return rows;
     }
+
+    private static InternalService.PDataRow getOneRow(List<Expr> row) throws UserException {
+        InternalService.PDataRow data = StmtExecutor.getRowStringValue(row, FormatOptions.getDefault());
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("add row: [{}]", data.getColList().stream().map(c -> c.getValue())
+                    .collect(Collectors.joining(",")));
+        }
+        return data;
+    }
+
+    private static List<InternalService.PDataRow> getRows(int targetColumnSize, List<Expr> rows) throws UserException {
+        List<InternalService.PDataRow> data = new ArrayList<>();
+        for (int i = 0; i < rows.size(); i += targetColumnSize) {
+            List<Expr> row = rows.subList(i, Math.min(i + targetColumnSize, rows.size()));
+            data.add(getOneRow(row));
+        }
+        return data;
+    }
+
+    // prepare command
+    public static void executeGroupCommitInsert(ConnectContext ctx, PreparedStatementContext preparedStmtCtx,
+            StatementContext statementContext) throws Exception {
+        PrepareCommand prepareCommand = preparedStmtCtx.command;
+        InsertIntoTableCommand command = (InsertIntoTableCommand) (prepareCommand.getLogicalPlan());
+        OlapTable table = (OlapTable) command.getTable(ctx);
+        for (int retry = 0; retry < MAX_RETRY; retry++) {
+            if (Env.getCurrentEnv().getGroupCommitManager().isBlock(table.getId())) {
+                String msg = "insert table " + table.getId() + SCHEMA_CHANGE;
+                LOG.info(msg);
+                throw new DdlException(msg);
+            }
+            boolean reuse = false;
+            GroupCommitPlanner groupCommitPlanner;
+            if (preparedStmtCtx.groupCommitPlanner.isPresent()
+                    && table.getBaseSchemaVersion() == preparedStmtCtx.groupCommitPlanner.get().baseSchemaVersion) {
+                groupCommitPlanner = preparedStmtCtx.groupCommitPlanner.get();
+                reuse = true;
+            } else {
+                // call nereids planner to check to sql
+                command.initPlan(ctx, new StmtExecutor(new ConnectContext(), ""), false);
+                List<String> targetColumnNames = command.getTargetColumns();
+                groupCommitPlanner = EnvFactory.getInstance()
+                        .createGroupCommitPlanner((Database) table.getDatabase(), table,
+                                targetColumnNames, ctx.queryId(),
+                                ConnectContext.get().getSessionVariable().getGroupCommit());
+                // TODO use planner column size
+                groupCommitPlanner.targetColumnSize = targetColumnNames == null ? table.getBaseSchema().size() :
+                        targetColumnNames.size();
+                preparedStmtCtx.groupCommitPlanner = Optional.of(groupCommitPlanner);
+            }
+            if (statementContext.getIdToPlaceholderRealExpr().size() % groupCommitPlanner.targetColumnSize != 0) {
+                throw new DdlException("Column size: " + statementContext.getIdToPlaceholderRealExpr().size()
+                        + " does not match with target column size: " + groupCommitPlanner.targetColumnSize);
+            }
+            List<Expr> valueExprs = statementContext.getIdToPlaceholderRealExpr().values().stream()
+                    .map(v -> ((Literal) v).toLegacyLiteral()).collect(Collectors.toList());
+            List<InternalService.PDataRow> rows = getRows(groupCommitPlanner.targetColumnSize, valueExprs);
+            PGroupCommitInsertResponse response = groupCommitPlanner.executeGroupCommitInsert(ctx, rows);
+            Pair<Boolean, Boolean> needRetryAndReplan = groupCommitPlanner.handleResponse(ctx, retry + 1 < MAX_RETRY,
+                    reuse, response);
+            if (needRetryAndReplan.first) {
+                if (needRetryAndReplan.second) {
+                    preparedStmtCtx.groupCommitPlanner = Optional.empty();
+                }
+            } else {
+                break;
+            }
+        }
+    }
+
+    // return <need_retry, need_replan>
+    private Pair<Boolean, Boolean> handleResponse(ConnectContext ctx, boolean canRetry, boolean reuse,
+            PGroupCommitInsertResponse response) throws DdlException {
+        TStatusCode code = TStatusCode.findByValue(response.getStatus().getStatusCode());
+        ProtocolStringList errorMsgsList = response.getStatus().getErrorMsgsList();
+        if (canRetry && code != TStatusCode.OK && !errorMsgsList.isEmpty()) {
+            if (errorMsgsList.get(0).contains("schema version not match")) {
+                LOG.info("group commit insert failed. query: {}, db: {}, table: {}, schema version: {}, "
+                                + "backend: {}, status: {}", DebugUtil.printId(ctx.queryId()), db.getId(),
+                        table.getId(), baseSchemaVersion, backendId, response.getStatus());
+                return Pair.of(true, true);
+            } else if (errorMsgsList.get(0).contains("can not get a block queue")) {
+                return Pair.of(true, false);
+            }
+        }
+        if (code != TStatusCode.OK) {
+            handleInsertFailed(ctx, response);
+        } else {
+            setReturnInfo(ctx, reuse, response);
+        }
+        return Pair.of(false, false);
+    }
+
+    private void handleInsertFailed(ConnectContext ctx, PGroupCommitInsertResponse response) throws DdlException {
+        String errMsg = "group commit insert failed. db: " + db.getId() + ", table: " + table.getId()
+                + ", query: " + DebugUtil.printId(ctx.queryId()) + ", backend: " + backendId
+                + ", status: " + response.getStatus();
+        if (response.hasErrorUrl()) {
+            errMsg += ", error url: " + response.getErrorUrl();
+        }
+        ErrorReport.reportDdlException(errMsg.replaceAll("%", "%%"), ErrorCode.ERR_FAILED_WHEN_INSERT);
+    }
+
+    private void setReturnInfo(ConnectContext ctx, boolean reuse, PGroupCommitInsertResponse response) {
+        String labelName = response.getLabel();
+        TransactionStatus txnStatus = TransactionStatus.PREPARE;
+        long txnId = response.getTxnId();
+        long loadedRows = response.getLoadedRows();
+        long filteredRows = (int) response.getFilteredRows();
+        String errorUrl = response.getErrorUrl();
+        // the same as {@OlapInsertExecutor#setReturnInfo}
+        // {'label':'my_label1', 'status':'visible', 'txnId':'123'}
+        // {'label':'my_label1', 'status':'visible', 'txnId':'123' 'err':'error messages'}
+        StringBuilder sb = new StringBuilder();
+        sb.append("{'label':'").append(labelName).append("', 'status':'").append(txnStatus.name());
+        sb.append("', 'txnId':'").append(txnId).append("'");
+        if (table.getType() == TableType.MATERIALIZED_VIEW) {
+            sb.append("', 'rows':'").append(loadedRows).append("'");
+        }
+        /*if (!Strings.isNullOrEmpty(errMsg)) {
+            sb.append(", 'err':'").append(errMsg).append("'");
+        }*/
+        if (!Strings.isNullOrEmpty(errorUrl)) {
+            sb.append(", 'err_url':'").append(errorUrl).append("'");
+        }
+        sb.append(", 'query_id':'").append(DebugUtil.printId(ctx.queryId())).append("'");
+        if (reuse) {
+            sb.append(", 'reuse_group_commit_plan':'").append(true).append("'");
+        }
+        sb.append("}");
+
+        ctx.getState().setOk(loadedRows, (int) filteredRows, sb.toString());
+        // set insert result in connection context,
+        // so that user can use `show insert result` to get info of the last insert operation.
+        ctx.setOrUpdateInsertResult(txnId, labelName, db.getFullName(), table.getName(),
+                txnStatus, loadedRows, (int) filteredRows);
+        // update it, so that user can get loaded rows in fe.audit.log
+        ctx.updateReturnRows((int) loadedRows);
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/PreparedStatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/PreparedStatementContext.java
index 8decad799171632..0174befee5b9d09 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/PreparedStatementContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/PreparedStatementContext.java
@@ -19,6 +19,7 @@
 
 import org.apache.doris.nereids.StatementContext;
 import org.apache.doris.nereids.trees.plans.commands.PrepareCommand;
+import org.apache.doris.planner.GroupCommitPlanner;
 
 import java.util.Optional;
 
@@ -28,6 +29,7 @@ public class PreparedStatementContext {
     StatementContext statementContext;
     public String stmtString;
     public Optional<ShortCircuitQueryContext> shortCircuitQueryContext = Optional.empty();
+    public Optional<GroupCommitPlanner> groupCommitPlanner = Optional.empty();
 
     // Timestamp in millisecond last command starts at
     protected volatile long startTime;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 0e20ed23f2c0759..8f35ef4cb9da69b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -147,6 +147,7 @@ public class SessionVariable implements Serializable, Writable {
 
     public static final String ENABLE_SERVER_SIDE_PREPARED_STATEMENT = "enable_server_side_prepared_statement";
     public static final String MAX_PREPARED_STMT_COUNT = "max_prepared_stmt_count";
+    public static final String ENABLE_GROUP_COMMIT_FULL_PREPARE = "enable_group_commit_full_prepare";
     public static final String PREFER_JOIN_METHOD = "prefer_join_method";
 
     public static final String ENABLE_FOLD_CONSTANT_BY_BE = "enable_fold_constant_by_be";
@@ -1640,6 +1641,9 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) {
                 "服务端prepared statement最大个数", "the maximum prepared statements server holds."})
     public int maxPreparedStmtCount = 100000;
 
+    @VariableMgr.VarAttr(name = ENABLE_GROUP_COMMIT_FULL_PREPARE)
+    public boolean enableGroupCommitFullPrepare = true;
+
     // Default value is false, which means the group by and having clause
     // should first use column name not alias. According to mysql.
     @VariableMgr.VarAttr(name = GROUP_BY_AND_HAVING_USE_ALIAS_FIRST, varType = VariableAnnotation.DEPRECATED)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index 021e0a58cd96906..f693f5b82aac29a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -725,7 +725,7 @@ private void executeByNereids(TUniqueId queryId) throws Exception {
         }
         if (logicalPlan instanceof Command) {
             if (logicalPlan instanceof Redirect) {
-                OlapGroupCommitInsertExecutor.analyzeGroupCommit(logicalPlan);
+                OlapGroupCommitInsertExecutor.analyzeGroupCommit(context, logicalPlan);
                 redirectStatus = ((Redirect) logicalPlan).toRedirectStatus();
                 if (isForwardToMaster()) {
                     // before forward to master, we also need to set profileType in this node
@@ -2391,7 +2391,7 @@ private void handleInsertStmt() throws Exception {
                     LOG.info("group commit insert failed. stmt: {}, query_id: {}, db_id: {}, table_id: {}"
                                     + ", schema version: {}, backend_id: {}, status: {}, retry: {}",
                             insertStmt.getOrigStmt().originStmt, DebugUtil.printId(context.queryId()), dbId, tableId,
-                            nativeInsertStmt.getBaseSchemaVersion(), groupCommitPlanner.getBackend().getId(),
+                            nativeInsertStmt.getBaseSchemaVersion(), groupCommitPlanner.getBackendId(),
                             response.getStatus(), i);
                     if (i < maxRetry) {
                         List<TableIf> tables = Lists.newArrayList(insertStmt.getTargetTable());
@@ -2408,15 +2408,15 @@ private void handleInsertStmt() throws Exception {
                     } else {
                         errMsg = "group commit insert failed. db_id: " + dbId + ", table_id: " + tableId
                                 + ", query_id: " + DebugUtil.printId(context.queryId()) + ", backend_id: "
-                                + groupCommitPlanner.getBackend().getId() + ", status: " + response.getStatus();
+                                + groupCommitPlanner.getBackendId() + ", status: " + response.getStatus();
                         if (response.hasErrorUrl()) {
                             errMsg += ", error url: " + response.getErrorUrl();
                         }
                     }
                 } else if (code != TStatusCode.OK) {
                     errMsg = "group commit insert failed. db_id: " + dbId + ", table_id: " + tableId + ", query_id: "
-                            + DebugUtil.printId(context.queryId()) + ", backend_id: " + groupCommitPlanner.getBackend()
-                            .getId() + ", status: " + response.getStatus();
+                            + DebugUtil.printId(context.queryId()) + ", backend_id: "
+                            + groupCommitPlanner.getBackendId() + ", status: " + response.getStatus();
                     if (response.hasErrorUrl()) {
                         errMsg += ", error url: " + response.getErrorUrl();
                     }
diff --git a/regression-test/suites/insert_p0/insert_group_commit_with_exception.groovy b/regression-test/suites/insert_p0/insert_group_commit_with_exception.groovy
index 7a22039a41cbe99..7b1a35eddf03f05 100644
--- a/regression-test/suites/insert_p0/insert_group_commit_with_exception.groovy
+++ b/regression-test/suites/insert_p0/insert_group_commit_with_exception.groovy
@@ -117,7 +117,6 @@ suite("insert_group_commit_with_exception") {
             Statement statement = connection.createStatement();
             statement.execute("use ${db}");
             statement.execute("set group_commit = sync_mode");
-            statement.execute("set enable_server_side_prepared_statement = true")
             // without column
             try (PreparedStatement ps = connection.prepareStatement("insert into ${table} values(?, ?, ?, ?)")) {
                 ps.setObject(1, 8);
diff --git a/regression-test/suites/insert_p0/insert_group_commit_with_prepare_stmt.groovy b/regression-test/suites/insert_p0/insert_group_commit_with_prepare_stmt.groovy
index e93e157aa5d7af0..44e8d173ca76ecc 100644
--- a/regression-test/suites/insert_p0/insert_group_commit_with_prepare_stmt.groovy
+++ b/regression-test/suites/insert_p0/insert_group_commit_with_prepare_stmt.groovy
@@ -89,7 +89,9 @@ suite("insert_group_commit_with_prepare_stmt") {
             assertTrue(serverInfo.contains("'status':'PREPARE'"))
             assertTrue(serverInfo.contains("'label':'group_commit_"))
             // TODO: currently if enable_server_side_prepared_statement = true, will not reuse plan
-            // assertEquals(reuse_plan, serverInfo.contains("reuse_group_commit_plan"))
+            if (reuse_plan) {
+                assertEquals(reuse_plan, serverInfo.contains("reuse_group_commit_plan"))
+            }
         } else {
             // for batch insert
             ConnectionImpl connection = (ConnectionImpl) stmt.getConnection()
diff --git a/regression-test/suites/insert_p0/transaction/txn_insert_inject_case.groovy b/regression-test/suites/insert_p0/transaction/txn_insert_inject_case.groovy
index 347f99c20042777..65556a9130109ad 100644
--- a/regression-test/suites/insert_p0/transaction/txn_insert_inject_case.groovy
+++ b/regression-test/suites/insert_p0/transaction/txn_insert_inject_case.groovy
@@ -134,7 +134,7 @@ suite("txn_insert_inject_case", "nonConcurrent") {
         }
 
         def result = sql "SELECT COUNT(*) FROM ${table}_0"
-        rowCount = result[0][0]
+        def rowCount = result[0][0]
         assertEquals(0, rowCount)
         // sleep(10000)
     } finally {

From c50bd335a68462082418dbc7b7c079d3dc2edc8c Mon Sep 17 00:00:00 2001
From: Pxl <xl@selectdb.com>
Date: Thu, 9 Jan 2025 12:28:05 +0800
Subject: [PATCH 067/140] [Bug](explain) fix wrong display of SortAlgorithm on
 explain (#46559)

### What problem does this PR solve?
 fix wrong display of SortAlgorithm on explain
---
 be/src/pipeline/exec/sort_sink_operator.cpp   |   5 +
 .../apache/doris/planner/AnalyticPlanner.java |   1 -
 .../org/apache/doris/planner/PlanNode.java    |   7 --
 .../doris/planner/SingleNodePlanner.java      |   1 -
 .../org/apache/doris/planner/SortNode.java    | 100 ++++++++----------
 5 files changed, 51 insertions(+), 63 deletions(-)

diff --git a/be/src/pipeline/exec/sort_sink_operator.cpp b/be/src/pipeline/exec/sort_sink_operator.cpp
index 6bec42ac62d1921..c24560a9a32a216 100644
--- a/be/src/pipeline/exec/sort_sink_operator.cpp
+++ b/be/src/pipeline/exec/sort_sink_operator.cpp
@@ -72,6 +72,11 @@ Status SortSinkLocalState::open(RuntimeState* state) {
     _shared_state->sorter->init_profile(_profile);
 
     _profile->add_info_string("TOP-N", p._limit == -1 ? "false" : "true");
+    _profile->add_info_string(
+            "SortAlgorithm",
+            p._algorithm == TSortAlgorithm::HEAP_SORT
+                    ? "HEAP_SORT"
+                    : (p._algorithm == TSortAlgorithm::TOPN_SORT ? "TOPN_SORT" : "FULL_SORT"));
     return Status::OK();
 }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticPlanner.java
index 7687b5812a459cb..3b29e8afb43535a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticPlanner.java
@@ -413,7 +413,6 @@ private PlanNode createSortGroupPlan(PlanNode root, SortGroup sortGroup,
 
             SortInfo sortInfo = createSortInfo(newRoot, sortExprs, isAsc, nullsFirst);
             SortNode sortNode = new SortNode(ctx.getNextNodeId(), newRoot, sortInfo, false);
-            sortNode.setDefaultLimit(false);
 
             // if this sort group does not have partitioning exprs, we want the sort
             // to be executed like a regular distributed sort
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
index c83cd9cde99b999..c6b06e7576c97ab 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
@@ -322,13 +322,6 @@ public void setOffset(long offset) {
         this.offset = offset;
     }
 
-    /**
-     * Used by new optimizer only.
-     */
-    public void setOffSetDirectly(long offset) {
-        this.offset = offset;
-    }
-
     public boolean hasLimit() {
         return limit > -1;
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
index f29ff4dba32b9ac..1ec738eba479413 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
@@ -313,7 +313,6 @@ private PlanNode createQueryPlan(QueryStmt stmt, Analyzer analyzer, long default
             boolean useTopN = true;
             root = new SortNode(ctx.getNextNodeId(), root, stmt.getSortInfo(),
                     useTopN);
-            ((SortNode) root).setDefaultLimit(limit == -1);
             root.setOffset(stmt.getOffset());
             if (useTopN) {
                 if (sqlSelectLimit >= 0) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java
index e3eb08c3e751038..2bfa76eb513dd65 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java
@@ -71,11 +71,11 @@ public class SortNode extends PlanNode {
     // exchange node, and the sort node is used for the ORDER BY .
     private boolean mergeByexchange = false;
 
-    private boolean isDefaultLimit;
     // if true, the output of this node feeds an AnalyticNode
     private boolean isAnalyticSort;
     private boolean isColocate = false;
     private DataPartition inputPartition;
+    TSortAlgorithm algorithm;
 
     private boolean isUnusedExprRemoved = false;
 
@@ -84,42 +84,49 @@ public class SortNode extends PlanNode {
     /**
      * Constructor.
      */
-    public SortNode(PlanNodeId id, PlanNode input, SortInfo info, boolean useTopN,
-            boolean isDefaultLimit, long offset) {
+    public SortNode(PlanNodeId id, PlanNode input, SortInfo info, boolean useTopN, long offset) {
         super(id, useTopN ? "TOP-N" : "SORT", StatisticalType.SORT_NODE);
         this.info = info;
         this.useTopN = useTopN;
-        this.isDefaultLimit = isDefaultLimit;
         this.tupleIds.addAll(Lists.newArrayList(info.getSortTupleDescriptor().getId()));
         this.tblRefIds.addAll(Lists.newArrayList(info.getSortTupleDescriptor().getId()));
         this.nullableTupleIds.addAll(input.getNullableTupleIds());
         this.children.add(input);
         this.offset = offset;
         Preconditions.checkArgument(info.getOrderingExprs().size() == info.getIsAscOrder().size());
+        updateSortAlgorithm();
     }
 
     public SortNode(PlanNodeId id, PlanNode input, SortInfo info, boolean useTopN) {
-        this(id, input, info, useTopN, true, 0);
+        this(id, input, info, useTopN, 0);
     }
 
-    /**
-     * Clone 'inputSortNode' for distributed Top-N.
-     */
-    public SortNode(PlanNodeId id, SortNode inputSortNode, PlanNode child) {
-        super(id, inputSortNode, inputSortNode.useTopN ? "TOP-N" : "SORT", StatisticalType.SORT_NODE);
-        this.info = inputSortNode.info;
-        this.useTopN = inputSortNode.useTopN;
-        this.isDefaultLimit = inputSortNode.isDefaultLimit;
-        this.children.add(child);
-        this.offset = inputSortNode.offset;
-    }
-
-    /**
-     * set isDefaultLimit when translate PhysicalLimit
-     * @param defaultLimit
-     */
-    public void setDefaultLimit(boolean defaultLimit) {
-        isDefaultLimit = defaultLimit;
+    private void updateSortAlgorithm() {
+        ConnectContext connectContext = ConnectContext.get();
+        if (connectContext != null && !connectContext.getSessionVariable().forceSortAlgorithm.isEmpty()) {
+            String algo = connectContext.getSessionVariable().forceSortAlgorithm;
+            if (algo.equals("heap")) {
+                algorithm = TSortAlgorithm.HEAP_SORT;
+            } else if (algo.equals("topn")) {
+                algorithm = TSortAlgorithm.TOPN_SORT;
+            } else {
+                algorithm = TSortAlgorithm.FULL_SORT;
+            }
+        } else {
+            if (limit <= 0) {
+                algorithm = TSortAlgorithm.FULL_SORT;
+            } else if (hasRuntimePredicate || useTwoPhaseReadOpt) {
+                algorithm = TSortAlgorithm.HEAP_SORT;
+            } else {
+                if (limit + offset < 50000) {
+                    algorithm = TSortAlgorithm.HEAP_SORT;
+                } else if (limit + offset < 20000000) {
+                    algorithm = TSortAlgorithm.FULL_SORT;
+                } else {
+                    algorithm = TSortAlgorithm.TOPN_SORT;
+                }
+            }
+        }
     }
 
     public void setIsAnalyticSort(boolean v) {
@@ -160,6 +167,7 @@ public boolean getUseTwoPhaseReadOpt() {
 
     public void setUseTwoPhaseReadOpt(boolean useTwoPhaseReadOpt) {
         this.useTwoPhaseReadOpt = useTwoPhaseReadOpt;
+        updateSortAlgorithm();
     }
 
     public List<Expr> getResolvedTupleExprs() {
@@ -201,12 +209,9 @@ public String getNodeExplainString(String detailPrefix, TExplainLevel detailLeve
         }
 
         output.append(detailPrefix + "algorithm: ");
-        boolean isFixedLength = info.getOrderingExprs().stream().allMatch(e -> !e.getType().isStringType()
-                && !e.getType().isCollectionType());
-        if (limit > 0 && limit + offset < 1024 && (useTwoPhaseReadOpt || hasRuntimePredicate
-                || isFixedLength)) {
+        if (algorithm == TSortAlgorithm.HEAP_SORT) {
             output.append("heap sort\n");
-        } else if (limit > 0 && !isFixedLength && limit + offset < 256) {
+        } else if (algorithm == TSortAlgorithm.TOPN_SORT) {
             output.append("topn sort\n");
         } else {
             output.append("full sort\n");
@@ -340,32 +345,7 @@ protected void toThrift(TPlanNode msg) {
         msg.sort_node.setIsAnalyticSort(isAnalyticSort);
         msg.sort_node.setIsColocate(isColocate);
 
-        ConnectContext connectContext = ConnectContext.get();
-        TSortAlgorithm algorithm;
-        if (connectContext != null && !connectContext.getSessionVariable().forceSortAlgorithm.isEmpty()) {
-            String algo = connectContext.getSessionVariable().forceSortAlgorithm;
-            if (algo.equals("heap")) {
-                algorithm = TSortAlgorithm.HEAP_SORT;
-            } else if (algo.equals("topn")) {
-                algorithm = TSortAlgorithm.TOPN_SORT;
-            } else {
-                algorithm = TSortAlgorithm.FULL_SORT;
-            }
-        } else {
-            if (limit <= 0) {
-                algorithm = TSortAlgorithm.FULL_SORT;
-            } else if (hasRuntimePredicate || useTwoPhaseReadOpt) {
-                algorithm = TSortAlgorithm.HEAP_SORT;
-            } else {
-                if (limit + offset < 50000) {
-                    algorithm = TSortAlgorithm.HEAP_SORT;
-                } else if (limit + offset < 20000000) {
-                    algorithm = TSortAlgorithm.FULL_SORT;
-                } else {
-                    algorithm = TSortAlgorithm.TOPN_SORT;
-                }
-            }
-        }
+
         msg.sort_node.setAlgorithm(algorithm);
     }
 
@@ -401,5 +381,17 @@ public void setColocate(boolean colocate) {
 
     public void setHasRuntimePredicate() {
         this.hasRuntimePredicate = true;
+        updateSortAlgorithm();
+    }
+
+    @Override
+    public void setLimit(long limit) {
+        super.setLimit(limit);
+        updateSortAlgorithm();
+    }
+
+    public void setOffset(long offset) {
+        super.setOffset(offset);
+        updateSortAlgorithm();
     }
 }

From ebd01c93356faedd14a8a3b2b884136677274ffa Mon Sep 17 00:00:00 2001
From: shuke <shuke@selectdb.com>
Date: Thu, 9 Jan 2025 12:45:08 +0800
Subject: [PATCH 068/140] =?UTF-8?q?[regression-test](fix)=20move=20set=5Fa?=
 =?UTF-8?q?nd=5Funset=5Fvariable=5Fcommand.groovy=20to=20=E2=80=A6=20(#466?=
 =?UTF-8?q?67)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…nonConcurrent
---
 .../set_and_unset_variable_command.groovy                       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/regression-test/suites/nereids_p0/ddl/set_and_unset_commands/set_and_unset_variable_command.groovy b/regression-test/suites/nereids_p0/ddl/set_and_unset_commands/set_and_unset_variable_command.groovy
index 1eae1ad77a46f33..46e253cd085b8f1 100644
--- a/regression-test/suites/nereids_p0/ddl/set_and_unset_commands/set_and_unset_variable_command.groovy
+++ b/regression-test/suites/nereids_p0/ddl/set_and_unset_commands/set_and_unset_variable_command.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("set_and_unset_variable_command") {
+suite("set_and_unset_variable_command", "nonConcurrent") {
     sql "set enable_nereids_planner=true"
     sql "set enable_fallback_to_original_planner=false"
 

From 284fed06fa32e650369f50f9d46c132305ace1c5 Mon Sep 17 00:00:00 2001
From: Dongyang Li <lidongyang@selectdb.com>
Date: Thu, 9 Jan 2025 14:20:29 +0800
Subject: [PATCH 069/140] [chore](ci) clean fdb all kv (#46665)

---
 regression-test/pipeline/common/doris-utils.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/regression-test/pipeline/common/doris-utils.sh b/regression-test/pipeline/common/doris-utils.sh
index 27f15fd192f75c5..f1adc5d99e1f7fb 100644
--- a/regression-test/pipeline/common/doris-utils.sh
+++ b/regression-test/pipeline/common/doris-utils.sh
@@ -227,6 +227,7 @@ function clean_fdb() {
         fdbcli --exec "writemode on;clearrange \x01\x10recycle\x00\x01\x10${instance_id}\x00\x01 \x01\x10recycle\x00\x01\x10${instance_id}\x00\xff\x00\x01" &&
         fdbcli --exec "writemode on;clearrange \x01\x10job\x00\x01\x10${instance_id}\x00\x01 \x01\x10job\x00\x01\x10${instance_id}\x00\xff\x00\x01" &&
         fdbcli --exec "writemode on;clearrange \x01\x10copy\x00\x01\x10${instance_id}\x00\x01 \x01\x10copy\x00\x01\x10${instance_id}\x00\xff\x00\x01" &&
+        fdbcli --exec "writemode on;clearrange \x00 \xff" &&
         rm -f /var/log/foundationdb/*; then
         echo "INFO: fdb cleaned."
     else

From c2212bf97febaea921e8f32e2e13d42807a2dd12 Mon Sep 17 00:00:00 2001
From: Dongyang Li <lidongyang@selectdb.com>
Date: Thu, 9 Jan 2025 14:21:07 +0800
Subject: [PATCH 070/140] [fix](case) add nonConcurrent   to out file path
 (#46679)

### What problem does this PR solve?
fix Missing outputFile

Issue Number: close #xxx

Related PR: #46062
---
 .../sql/affinityByIssuesAndPRs1.out                               | 0
 .../sql/affinityByIssuesAndPRs2.out                               | 0
 .../sql/authorsWithTheMostPushes.out                              | 0
 .../sql/countingStar1.out                                         | 0
 .../sql/countingStar2.out                                         | 0
 .../sql/countingStar3.out                                         | 0
 .../sql/distributionOfRepositoriesByStarCount.out                 | 0
 .../sql/githubRoulette.out                                        | 0
 .../sql/howHasTheListOfTopRepositoriesChangedOverTheYears1.out    | 0
 .../sql/howHasTheListOfTopRepositoriesChangedOverTheYears2.out    | 0
 .../sql/howHasTheListOfTopRepositoriesChangedOverTheYears3.out    | 0
 .../sql/howHasTheListOfTopRepositoriesChangedOverTheYears4.out    | 0
 .../sql/howHasTheListOfTopRepositoriesChangedOverTheYears5.out    | 0
 .../sql/howHasTheListOfTopRepositoriesChangedOverTheYears6.out    | 0
 .../sql/howHasTheTotalNumberOfStarsChangedOverTime.out            | 0
 .../sql/issuesWithTheMostComments1.out                            | 0
 .../sql/issuesWithTheMostComments2.out                            | 0
 .../sql/issuesWithTheMostComments3.out                            | 0
 .../sql/issuesWithTheMostComments4.out                            | 0
 .../sql/issuesWithTheMostComments5.out                            | 0
 .../sql/issuesWithTheMostComments6.out                            | 0
 .../sql/issuesWithTheMostComments7.out                            | 0
 .../sql/load.out                                                  | 0
 .../sql/mostForkedRepositories.out                                | 0
 .../sql/mostPopularCommentsOnGithub.out                           | 0
 .../sql/organizationsByTheNumberOfRepositories.out                | 0
 .../sql/organizationsByTheNumberOfStars.out                       | 0
 .../sql/proportionsBetweenStarsAndForks1.out                      | 0
 .../sql/proportionsBetweenStarsAndForks2.out                      | 0
 .../sql/proportionsBetweenStarsAndForks3.out                      | 0
 .../sql/proportionsBetweenStarsAndForks4.out                      | 0
 .../sql/proportionsBetweenStarsAndForks5.out                      | 0
 .../sql/repositoriesByAmountOfModifiedCode.out                    | 0
 .../sql/repositoriesByTheNumberOfPushes.out                       | 0
 .../sql/repositoriesWithClickhouse_related_comments1.out          | 0
 .../sql/repositoriesWithClickhouse_related_comments2.out          | 0
 .../sql/repositoriesWithDoris_related_comments1.out               | 0
 .../sql/repositoriesWithDoris_related_comments2.out               | 0
 .../sql/repositoriesWithTheHighestGrowthYoY.out                   | 0
 .../sql/repositoriesWithTheMaximumAmountOfIssues1.out             | 0
 .../sql/repositoriesWithTheMaximumAmountOfIssues2.out             | 0
 .../sql/repositoriesWithTheMaximumAmountOfIssues3.out             | 0
 .../sql/repositoriesWithTheMaximumAmountOfIssues4.out             | 0
 .../sql/repositoriesWithTheMaximumAmountOfPullRequests1.out       | 0
 .../sql/repositoriesWithTheMaximumAmountOfPullRequests2.out       | 0
 .../sql/repositoriesWithTheMaximumNumberOfAcceptedInvitations.out | 0
 .../sql/repositoriesWithTheMostPeopleWhoHavePushAccess1.out       | 0
 .../sql/repositoriesWithTheMostStarsOverOneDay1.out               | 0
 .../sql/repositoriesWithTheMostSteadyGrowthOverTime.out           | 0
 .../sql/repositoriesWithTheWorstStagnation_order.out              | 0
 .../sql/repositoryAffinityList1.out                               | 0
 .../sql/repositoryAffinityList2.out                               | 0
 .../sql/starsFromHeavyGithubUsers1.out                            | 0
 .../sql/starsFromHeavyGithubUsers2.out                            | 0
 .../sql/theLongestRepositoryNames1.out                            | 0
 .../sql/theLongestRepositoryNames2.out                            | 0
 .../sql/theMostToughCodeReviews.out                               | 0
 .../sql/theTotalNumberOfRepositoriesOnGithub.out                  | 0
 .../sql/theTotalNumberOfUsersOnGithub1.out                        | 0
 .../sql/theTotalNumberOfUsersOnGithub2.out                        | 0
 .../sql/theTotalNumberOfUsersOnGithub3.out                        | 0
 .../sql/theTotalNumberOfUsersOnGithub4.out                        | 0
 .../sql/topRepositoriesByStars.out                                | 0
 .../sql/whatIsTheBestDayOfTheWeekToCatchAStar.out                 | 0
 .../sql/whoAreAllThosePeopleGivingStars1.out                      | 0
 .../sql/whoAreAllThosePeopleGivingStars2.out                      | 0
 .../sql/whoAreAllThosePeopleGivingStars3.out                      | 0
 67 files changed, 0 insertions(+), 0 deletions(-)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/affinityByIssuesAndPRs1.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/affinityByIssuesAndPRs2.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/authorsWithTheMostPushes.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/countingStar1.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/countingStar2.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/countingStar3.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/distributionOfRepositoriesByStarCount.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/githubRoulette.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/howHasTheListOfTopRepositoriesChangedOverTheYears1.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/howHasTheListOfTopRepositoriesChangedOverTheYears2.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/howHasTheListOfTopRepositoriesChangedOverTheYears3.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/howHasTheListOfTopRepositoriesChangedOverTheYears4.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/howHasTheListOfTopRepositoriesChangedOverTheYears5.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/howHasTheListOfTopRepositoriesChangedOverTheYears6.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/howHasTheTotalNumberOfStarsChangedOverTime.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/issuesWithTheMostComments1.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/issuesWithTheMostComments2.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/issuesWithTheMostComments3.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/issuesWithTheMostComments4.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/issuesWithTheMostComments5.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/issuesWithTheMostComments6.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/issuesWithTheMostComments7.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/load.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/mostForkedRepositories.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/mostPopularCommentsOnGithub.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/organizationsByTheNumberOfRepositories.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/organizationsByTheNumberOfStars.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/proportionsBetweenStarsAndForks1.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/proportionsBetweenStarsAndForks2.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/proportionsBetweenStarsAndForks3.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/proportionsBetweenStarsAndForks4.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/proportionsBetweenStarsAndForks5.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/repositoriesByAmountOfModifiedCode.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/repositoriesByTheNumberOfPushes.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/repositoriesWithClickhouse_related_comments1.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/repositoriesWithClickhouse_related_comments2.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/repositoriesWithDoris_related_comments1.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/repositoriesWithDoris_related_comments2.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/repositoriesWithTheHighestGrowthYoY.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/repositoriesWithTheMaximumAmountOfIssues1.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/repositoriesWithTheMaximumAmountOfIssues2.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/repositoriesWithTheMaximumAmountOfIssues3.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/repositoriesWithTheMaximumAmountOfIssues4.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/repositoriesWithTheMaximumAmountOfPullRequests1.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/repositoriesWithTheMaximumAmountOfPullRequests2.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/repositoriesWithTheMaximumNumberOfAcceptedInvitations.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/repositoriesWithTheMostPeopleWhoHavePushAccess1.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/repositoriesWithTheMostStarsOverOneDay1.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/repositoriesWithTheMostSteadyGrowthOverTime.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/repositoriesWithTheWorstStagnation_order.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/repositoryAffinityList1.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/repositoryAffinityList2.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/starsFromHeavyGithubUsers1.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/starsFromHeavyGithubUsers2.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/theLongestRepositoryNames1.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/theLongestRepositoryNames2.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/theMostToughCodeReviews.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/theTotalNumberOfRepositoriesOnGithub.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/theTotalNumberOfUsersOnGithub1.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/theTotalNumberOfUsersOnGithub2.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/theTotalNumberOfUsersOnGithub3.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/theTotalNumberOfUsersOnGithub4.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/topRepositoriesByStars.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/whatIsTheBestDayOfTheWeekToCatchAStar.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/whoAreAllThosePeopleGivingStars1.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/whoAreAllThosePeopleGivingStars2.out (100%)
 rename regression-test/data/{variant_github_events_p2 => variant_github_events_nonConcurrent_p2}/sql/whoAreAllThosePeopleGivingStars3.out (100%)

diff --git a/regression-test/data/variant_github_events_p2/sql/affinityByIssuesAndPRs1.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/affinityByIssuesAndPRs1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/affinityByIssuesAndPRs1.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/affinityByIssuesAndPRs1.out
diff --git a/regression-test/data/variant_github_events_p2/sql/affinityByIssuesAndPRs2.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/affinityByIssuesAndPRs2.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/affinityByIssuesAndPRs2.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/affinityByIssuesAndPRs2.out
diff --git a/regression-test/data/variant_github_events_p2/sql/authorsWithTheMostPushes.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/authorsWithTheMostPushes.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/authorsWithTheMostPushes.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/authorsWithTheMostPushes.out
diff --git a/regression-test/data/variant_github_events_p2/sql/countingStar1.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/countingStar1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/countingStar1.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/countingStar1.out
diff --git a/regression-test/data/variant_github_events_p2/sql/countingStar2.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/countingStar2.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/countingStar2.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/countingStar2.out
diff --git a/regression-test/data/variant_github_events_p2/sql/countingStar3.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/countingStar3.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/countingStar3.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/countingStar3.out
diff --git a/regression-test/data/variant_github_events_p2/sql/distributionOfRepositoriesByStarCount.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/distributionOfRepositoriesByStarCount.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/distributionOfRepositoriesByStarCount.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/distributionOfRepositoriesByStarCount.out
diff --git a/regression-test/data/variant_github_events_p2/sql/githubRoulette.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/githubRoulette.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/githubRoulette.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/githubRoulette.out
diff --git a/regression-test/data/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears1.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears1.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears1.out
diff --git a/regression-test/data/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears2.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears2.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears2.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears2.out
diff --git a/regression-test/data/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears3.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears3.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears3.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears3.out
diff --git a/regression-test/data/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears4.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears4.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears4.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears4.out
diff --git a/regression-test/data/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears5.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears5.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears5.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears5.out
diff --git a/regression-test/data/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears6.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears6.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears6.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears6.out
diff --git a/regression-test/data/variant_github_events_p2/sql/howHasTheTotalNumberOfStarsChangedOverTime.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/howHasTheTotalNumberOfStarsChangedOverTime.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/howHasTheTotalNumberOfStarsChangedOverTime.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/howHasTheTotalNumberOfStarsChangedOverTime.out
diff --git a/regression-test/data/variant_github_events_p2/sql/issuesWithTheMostComments1.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/issuesWithTheMostComments1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/issuesWithTheMostComments1.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/issuesWithTheMostComments1.out
diff --git a/regression-test/data/variant_github_events_p2/sql/issuesWithTheMostComments2.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/issuesWithTheMostComments2.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/issuesWithTheMostComments2.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/issuesWithTheMostComments2.out
diff --git a/regression-test/data/variant_github_events_p2/sql/issuesWithTheMostComments3.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/issuesWithTheMostComments3.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/issuesWithTheMostComments3.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/issuesWithTheMostComments3.out
diff --git a/regression-test/data/variant_github_events_p2/sql/issuesWithTheMostComments4.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/issuesWithTheMostComments4.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/issuesWithTheMostComments4.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/issuesWithTheMostComments4.out
diff --git a/regression-test/data/variant_github_events_p2/sql/issuesWithTheMostComments5.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/issuesWithTheMostComments5.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/issuesWithTheMostComments5.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/issuesWithTheMostComments5.out
diff --git a/regression-test/data/variant_github_events_p2/sql/issuesWithTheMostComments6.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/issuesWithTheMostComments6.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/issuesWithTheMostComments6.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/issuesWithTheMostComments6.out
diff --git a/regression-test/data/variant_github_events_p2/sql/issuesWithTheMostComments7.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/issuesWithTheMostComments7.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/issuesWithTheMostComments7.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/issuesWithTheMostComments7.out
diff --git a/regression-test/data/variant_github_events_p2/sql/load.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/load.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/load.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/load.out
diff --git a/regression-test/data/variant_github_events_p2/sql/mostForkedRepositories.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/mostForkedRepositories.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/mostForkedRepositories.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/mostForkedRepositories.out
diff --git a/regression-test/data/variant_github_events_p2/sql/mostPopularCommentsOnGithub.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/mostPopularCommentsOnGithub.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/mostPopularCommentsOnGithub.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/mostPopularCommentsOnGithub.out
diff --git a/regression-test/data/variant_github_events_p2/sql/organizationsByTheNumberOfRepositories.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/organizationsByTheNumberOfRepositories.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/organizationsByTheNumberOfRepositories.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/organizationsByTheNumberOfRepositories.out
diff --git a/regression-test/data/variant_github_events_p2/sql/organizationsByTheNumberOfStars.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/organizationsByTheNumberOfStars.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/organizationsByTheNumberOfStars.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/organizationsByTheNumberOfStars.out
diff --git a/regression-test/data/variant_github_events_p2/sql/proportionsBetweenStarsAndForks1.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/proportionsBetweenStarsAndForks1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/proportionsBetweenStarsAndForks1.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/proportionsBetweenStarsAndForks1.out
diff --git a/regression-test/data/variant_github_events_p2/sql/proportionsBetweenStarsAndForks2.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/proportionsBetweenStarsAndForks2.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/proportionsBetweenStarsAndForks2.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/proportionsBetweenStarsAndForks2.out
diff --git a/regression-test/data/variant_github_events_p2/sql/proportionsBetweenStarsAndForks3.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/proportionsBetweenStarsAndForks3.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/proportionsBetweenStarsAndForks3.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/proportionsBetweenStarsAndForks3.out
diff --git a/regression-test/data/variant_github_events_p2/sql/proportionsBetweenStarsAndForks4.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/proportionsBetweenStarsAndForks4.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/proportionsBetweenStarsAndForks4.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/proportionsBetweenStarsAndForks4.out
diff --git a/regression-test/data/variant_github_events_p2/sql/proportionsBetweenStarsAndForks5.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/proportionsBetweenStarsAndForks5.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/proportionsBetweenStarsAndForks5.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/proportionsBetweenStarsAndForks5.out
diff --git a/regression-test/data/variant_github_events_p2/sql/repositoriesByAmountOfModifiedCode.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesByAmountOfModifiedCode.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/repositoriesByAmountOfModifiedCode.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesByAmountOfModifiedCode.out
diff --git a/regression-test/data/variant_github_events_p2/sql/repositoriesByTheNumberOfPushes.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesByTheNumberOfPushes.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/repositoriesByTheNumberOfPushes.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesByTheNumberOfPushes.out
diff --git a/regression-test/data/variant_github_events_p2/sql/repositoriesWithClickhouse_related_comments1.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithClickhouse_related_comments1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/repositoriesWithClickhouse_related_comments1.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithClickhouse_related_comments1.out
diff --git a/regression-test/data/variant_github_events_p2/sql/repositoriesWithClickhouse_related_comments2.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithClickhouse_related_comments2.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/repositoriesWithClickhouse_related_comments2.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithClickhouse_related_comments2.out
diff --git a/regression-test/data/variant_github_events_p2/sql/repositoriesWithDoris_related_comments1.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithDoris_related_comments1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/repositoriesWithDoris_related_comments1.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithDoris_related_comments1.out
diff --git a/regression-test/data/variant_github_events_p2/sql/repositoriesWithDoris_related_comments2.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithDoris_related_comments2.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/repositoriesWithDoris_related_comments2.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithDoris_related_comments2.out
diff --git a/regression-test/data/variant_github_events_p2/sql/repositoriesWithTheHighestGrowthYoY.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithTheHighestGrowthYoY.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/repositoriesWithTheHighestGrowthYoY.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithTheHighestGrowthYoY.out
diff --git a/regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfIssues1.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithTheMaximumAmountOfIssues1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfIssues1.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithTheMaximumAmountOfIssues1.out
diff --git a/regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfIssues2.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithTheMaximumAmountOfIssues2.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfIssues2.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithTheMaximumAmountOfIssues2.out
diff --git a/regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfIssues3.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithTheMaximumAmountOfIssues3.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfIssues3.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithTheMaximumAmountOfIssues3.out
diff --git a/regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfIssues4.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithTheMaximumAmountOfIssues4.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfIssues4.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithTheMaximumAmountOfIssues4.out
diff --git a/regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests1.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests1.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests1.out
diff --git a/regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests2.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests2.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests2.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests2.out
diff --git a/regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMaximumNumberOfAcceptedInvitations.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithTheMaximumNumberOfAcceptedInvitations.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMaximumNumberOfAcceptedInvitations.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithTheMaximumNumberOfAcceptedInvitations.out
diff --git a/regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess1.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess1.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess1.out
diff --git a/regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMostStarsOverOneDay1.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithTheMostStarsOverOneDay1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMostStarsOverOneDay1.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithTheMostStarsOverOneDay1.out
diff --git a/regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMostSteadyGrowthOverTime.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithTheMostSteadyGrowthOverTime.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/repositoriesWithTheMostSteadyGrowthOverTime.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithTheMostSteadyGrowthOverTime.out
diff --git a/regression-test/data/variant_github_events_p2/sql/repositoriesWithTheWorstStagnation_order.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithTheWorstStagnation_order.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/repositoriesWithTheWorstStagnation_order.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoriesWithTheWorstStagnation_order.out
diff --git a/regression-test/data/variant_github_events_p2/sql/repositoryAffinityList1.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoryAffinityList1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/repositoryAffinityList1.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoryAffinityList1.out
diff --git a/regression-test/data/variant_github_events_p2/sql/repositoryAffinityList2.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoryAffinityList2.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/repositoryAffinityList2.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/repositoryAffinityList2.out
diff --git a/regression-test/data/variant_github_events_p2/sql/starsFromHeavyGithubUsers1.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/starsFromHeavyGithubUsers1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/starsFromHeavyGithubUsers1.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/starsFromHeavyGithubUsers1.out
diff --git a/regression-test/data/variant_github_events_p2/sql/starsFromHeavyGithubUsers2.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/starsFromHeavyGithubUsers2.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/starsFromHeavyGithubUsers2.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/starsFromHeavyGithubUsers2.out
diff --git a/regression-test/data/variant_github_events_p2/sql/theLongestRepositoryNames1.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/theLongestRepositoryNames1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/theLongestRepositoryNames1.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/theLongestRepositoryNames1.out
diff --git a/regression-test/data/variant_github_events_p2/sql/theLongestRepositoryNames2.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/theLongestRepositoryNames2.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/theLongestRepositoryNames2.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/theLongestRepositoryNames2.out
diff --git a/regression-test/data/variant_github_events_p2/sql/theMostToughCodeReviews.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/theMostToughCodeReviews.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/theMostToughCodeReviews.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/theMostToughCodeReviews.out
diff --git a/regression-test/data/variant_github_events_p2/sql/theTotalNumberOfRepositoriesOnGithub.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/theTotalNumberOfRepositoriesOnGithub.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/theTotalNumberOfRepositoriesOnGithub.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/theTotalNumberOfRepositoriesOnGithub.out
diff --git a/regression-test/data/variant_github_events_p2/sql/theTotalNumberOfUsersOnGithub1.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/theTotalNumberOfUsersOnGithub1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/theTotalNumberOfUsersOnGithub1.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/theTotalNumberOfUsersOnGithub1.out
diff --git a/regression-test/data/variant_github_events_p2/sql/theTotalNumberOfUsersOnGithub2.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/theTotalNumberOfUsersOnGithub2.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/theTotalNumberOfUsersOnGithub2.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/theTotalNumberOfUsersOnGithub2.out
diff --git a/regression-test/data/variant_github_events_p2/sql/theTotalNumberOfUsersOnGithub3.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/theTotalNumberOfUsersOnGithub3.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/theTotalNumberOfUsersOnGithub3.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/theTotalNumberOfUsersOnGithub3.out
diff --git a/regression-test/data/variant_github_events_p2/sql/theTotalNumberOfUsersOnGithub4.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/theTotalNumberOfUsersOnGithub4.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/theTotalNumberOfUsersOnGithub4.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/theTotalNumberOfUsersOnGithub4.out
diff --git a/regression-test/data/variant_github_events_p2/sql/topRepositoriesByStars.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/topRepositoriesByStars.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/topRepositoriesByStars.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/topRepositoriesByStars.out
diff --git a/regression-test/data/variant_github_events_p2/sql/whatIsTheBestDayOfTheWeekToCatchAStar.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/whatIsTheBestDayOfTheWeekToCatchAStar.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/whatIsTheBestDayOfTheWeekToCatchAStar.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/whatIsTheBestDayOfTheWeekToCatchAStar.out
diff --git a/regression-test/data/variant_github_events_p2/sql/whoAreAllThosePeopleGivingStars1.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/whoAreAllThosePeopleGivingStars1.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/whoAreAllThosePeopleGivingStars1.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/whoAreAllThosePeopleGivingStars1.out
diff --git a/regression-test/data/variant_github_events_p2/sql/whoAreAllThosePeopleGivingStars2.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/whoAreAllThosePeopleGivingStars2.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/whoAreAllThosePeopleGivingStars2.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/whoAreAllThosePeopleGivingStars2.out
diff --git a/regression-test/data/variant_github_events_p2/sql/whoAreAllThosePeopleGivingStars3.out b/regression-test/data/variant_github_events_nonConcurrent_p2/sql/whoAreAllThosePeopleGivingStars3.out
similarity index 100%
rename from regression-test/data/variant_github_events_p2/sql/whoAreAllThosePeopleGivingStars3.out
rename to regression-test/data/variant_github_events_nonConcurrent_p2/sql/whoAreAllThosePeopleGivingStars3.out

From 7fc776fadc143a4deff1467cc9a467b1676f6399 Mon Sep 17 00:00:00 2001
From: lihangyu <lihangyu@selectdb.com>
Date: Thu, 9 Jan 2025 14:31:14 +0800
Subject: [PATCH 071/140] [Optimize](TabletMeta) Reuse TabletSchema in
 TabletMeta (#46572)

In clusters with a large number of tablets, the sheer volume of
TabletSchema instances can consume significant memory. This change
introduces TabletSchema reuse in TabletMeta to optimize memory usage.

With large number of tablets:
56G memory consumption reduced to 4G
---
 be/src/olap/tablet_meta.cpp | 18 +++++++++++++++++-
 be/src/olap/tablet_meta.h   |  2 ++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 43b0d5d8bd0ae03..e2f17730973b8d3 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -27,6 +27,7 @@
 #include <time.h>
 
 #include <cstdint>
+#include <memory>
 #include <set>
 #include <utility>
 
@@ -43,7 +44,9 @@
 #include "olap/olap_define.h"
 #include "olap/rowset/rowset.h"
 #include "olap/rowset/rowset_meta_manager.h"
+#include "olap/tablet_fwd.h"
 #include "olap/tablet_meta_manager.h"
+#include "olap/tablet_schema_cache.h"
 #include "olap/utils.h"
 #include "util/debug_points.h"
 #include "util/mem_info.h"
@@ -101,6 +104,12 @@ TabletMetaSharedPtr TabletMeta::create(
             request.time_series_compaction_level_threshold, inverted_index_file_storage_format);
 }
 
+TabletMeta::~TabletMeta() {
+    if (_handle) {
+        TabletSchemaCache::instance()->release(_handle);
+    }
+}
+
 TabletMeta::TabletMeta()
         : _tablet_uid(0, 0),
           _schema(new TabletSchema),
@@ -625,7 +634,14 @@ void TabletMeta::init_from_pb(const TabletMetaPB& tablet_meta_pb) {
     }
 
     // init _schema
-    _schema->init_from_pb(tablet_meta_pb.schema());
+    TabletSchemaSPtr schema = std::make_shared<TabletSchema>();
+    schema->init_from_pb(tablet_meta_pb.schema());
+    if (_handle) {
+        TabletSchemaCache::instance()->release(_handle);
+    }
+    auto pair = TabletSchemaCache::instance()->insert(schema->to_key());
+    _handle = pair.first;
+    _schema = pair.second;
 
     if (tablet_meta_pb.has_enable_unique_key_merge_on_write()) {
         _enable_unique_key_merge_on_write = tablet_meta_pb.enable_unique_key_merge_on_write();
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index 25f6bcd569be430..0cbc106c135610c 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -100,6 +100,7 @@ class TabletMeta : public MetadataAdder<TabletMeta> {
             const std::unordered_map<uint32_t, uint32_t>& col_ordinal_to_unique_id);
 
     TabletMeta();
+    ~TabletMeta() override;
     TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id, int64_t replica_id,
                int32_t schema_hash, int32_t shard_id, const TTabletSchema& tablet_schema,
                uint32_t next_unique_id,
@@ -321,6 +322,7 @@ class TabletMeta : public MetadataAdder<TabletMeta> {
     // the reference of _schema may use in tablet, so here need keep
     // the lifetime of tablemeta and _schema is same with tablet
     TabletSchemaSPtr _schema;
+    Cache::Handle* _handle = nullptr;
 
     std::vector<RowsetMetaSharedPtr> _rs_metas;
     // This variable _stale_rs_metas is used to record these rowsets‘ meta which are be compacted.

From ae1e6bc878ca4c02b924aa8d5f969c1b34fc48ba Mon Sep 17 00:00:00 2001
From: Yao-MR <yao.mr.cn@gmail.com>
Date: Thu, 9 Jan 2025 14:43:31 +0800
Subject: [PATCH 072/140] [Enhancement] (nereids)implement SHOW CATALOG COMMAND
 in nereids (#44633)

Issue Number: close #42745
---
 .../org/apache/doris/nereids/DorisParser.g4   |   4 +-
 .../apache/doris/datasource/CatalogMgr.java   |  31 ++++--
 .../nereids/parser/LogicalPlanBuilder.java    |  19 ++++
 .../doris/nereids/trees/plans/PlanType.java   |   1 +
 .../plans/commands/ShowCatalogCommand.java    | 104 ++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |   5 +
 .../nereids_p0/show/test_show_catalog.groovy  |  33 ++++++
 7 files changed, 184 insertions(+), 13 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCatalogCommand.java
 create mode 100644 regression-test/suites/nereids_p0/show/test_show_catalog.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 7e89accefca91be..7045733bafdba7e 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -273,6 +273,8 @@ supportedShowStatement
     | SHOW FILE ((FROM | IN) database=multipartIdentifier)?                         #showSmallFiles 
     | SHOW STORAGE? ENGINES                                                         #showStorageEngines
     | SHOW CREATE CATALOG name=identifier                                           #showCreateCatalog
+    | SHOW CATALOG name=identifier                                                  #showCatalog
+    | SHOW CATALOGS wildWhere?                                                      #showCatalogs
     | SHOW PROPERTY (FOR user=identifierOrText)? (LIKE STRING_LITERAL)?                         #showUserProperties
     | SHOW ALL PROPERTIES (LIKE STRING_LITERAL)?                                               #showAllProperties
     | SHOW COLLATION wildWhere?                                                     #showCollation
@@ -351,8 +353,6 @@ unsupportedShowStatement
         LEFT_PAREN functionArguments? RIGHT_PAREN
         ((FROM | IN) database=multipartIdentifier)?                                 #showCreateFunction
     | SHOW (DATABASES | SCHEMAS) (FROM catalog=identifier)? wildWhere?              #showDatabases
-    | SHOW CATALOGS wildWhere?                                                      #showCatalogs
-    | SHOW CATALOG name=identifier                                                  #showCatalog
     | SHOW FULL? (COLUMNS | FIELDS) (FROM | IN) tableName=multipartIdentifier
         ((FROM | IN) database=multipartIdentifier)? wildWhere?                      #showColumns
     | SHOW LOAD WARNINGS ((((FROM | IN) database=multipartIdentifier)?
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
index d73d2b85b262801..b3d561e518fe8d6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
@@ -405,14 +405,21 @@ public ShowResultSet showCatalogs(ShowCatalogStmt showStmt) throws AnalysisExcep
     }
 
     public ShowResultSet showCatalogs(ShowCatalogStmt showStmt, String currentCtlg) throws AnalysisException {
+        List<List<String>> rows = showCatalogs(showStmt.getCatalogName(), showStmt.getPattern(), currentCtlg);
+
+        return new ShowResultSet(showStmt.getMetaData(), rows);
+    }
+
+    public List<List<String>> showCatalogs(
+            String catalogName, String pattern, String currentCatalogName) throws AnalysisException {
         List<List<String>> rows = Lists.newArrayList();
         readLock();
         try {
-            if (showStmt.getCatalogName() == null) {
+            if (catalogName == null) {
                 PatternMatcher matcher = null;
-                if (showStmt.getPattern() != null) {
-                    matcher = PatternMatcherWrapper.createMysqlPattern(showStmt.getPattern(),
-                            CaseSensibility.CATALOG.getCaseSensibility());
+                if (pattern != null) {
+                    matcher = PatternMatcherWrapper.createMysqlPattern(pattern,
+                        CaseSensibility.CATALOG.getCaseSensibility());
                 }
                 for (CatalogIf catalog : listCatalogsWithCheckPriv(ConnectContext.get().getCurrentUserIdentity())) {
                     String name = catalog.getName();
@@ -424,7 +431,7 @@ public ShowResultSet showCatalogs(ShowCatalogStmt showStmt, String currentCtlg)
                     row.add(String.valueOf(catalog.getId()));
                     row.add(name);
                     row.add(catalog.getType());
-                    if (name.equals(currentCtlg)) {
+                    if (name.equals(currentCatalogName)) {
                         row.add("Yes");
                     } else {
                         row.add("No");
@@ -442,14 +449,16 @@ public ShowResultSet showCatalogs(ShowCatalogStmt showStmt, String currentCtlg)
                     });
                 }
             } else {
-                if (!nameToCatalog.containsKey(showStmt.getCatalogName())) {
-                    throw new AnalysisException("No catalog found with name: " + showStmt.getCatalogName());
+                if (!nameToCatalog.containsKey(catalogName)) {
+                    throw new AnalysisException("No catalog found with name: " + catalogName);
                 }
-                CatalogIf<DatabaseIf> catalog = nameToCatalog.get(showStmt.getCatalogName());
+                CatalogIf<DatabaseIf> catalog = nameToCatalog.get(catalogName);
                 if (!Env.getCurrentEnv().getAccessManager()
                         .checkCtlPriv(ConnectContext.get(), catalog.getName(), PrivPredicate.SHOW)) {
-                    ErrorReport.reportAnalysisException(ErrorCode.ERR_CATALOG_ACCESS_DENIED,
-                            ConnectContext.get().getQualifiedUser(), catalog.getName());
+                    ErrorReport.reportAnalysisException(
+                            ErrorCode.ERR_CATALOG_ACCESS_DENIED,
+                            ConnectContext.get().getQualifiedUser(),
+                            catalog.getName());
                 }
                 if (!Strings.isNullOrEmpty(catalog.getResource())) {
                     rows.add(Arrays.asList("resource", catalog.getResource()));
@@ -461,7 +470,7 @@ public ShowResultSet showCatalogs(ShowCatalogStmt showStmt, String currentCtlg)
             readUnlock();
         }
 
-        return new ShowResultSet(showStmt.getMetaData(), rows);
+        return rows;
     }
 
     public static Map<String, String> getCatalogPropertiesWithPrintable(CatalogIf<?> catalog) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 66ee9c33c093a89..b3a62e604c53c45 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -558,6 +558,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowAuthorsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowBackendsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowBrokerCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowCatalogCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCharsetCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCollationCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowConfigCommand;
@@ -4884,6 +4885,24 @@ public LogicalPlan visitShowCreateCatalog(ShowCreateCatalogContext ctx) {
         return new ShowCreateCatalogCommand(ctx.identifier().getText());
     }
 
+    @Override
+    public LogicalPlan visitShowCatalog(DorisParser.ShowCatalogContext ctx) {
+        return new ShowCatalogCommand(ctx.identifier().getText(), null);
+    }
+
+    @Override
+    public LogicalPlan visitShowCatalogs(DorisParser.ShowCatalogsContext ctx) {
+        String wild = null;
+        if (ctx.wildWhere() != null) {
+            if (ctx.wildWhere().LIKE() != null) {
+                wild = stripQuotes(ctx.wildWhere().STRING_LITERAL().getText());
+            } else if (ctx.wildWhere().WHERE() != null) {
+                wild = ctx.wildWhere().expression().getText();
+            }
+        }
+        return new ShowCatalogCommand(null, wild);
+    }
+
     @Override
     public LogicalPlan visitShowStorageEngines(ShowStorageEnginesContext ctx) {
         return new ShowStorageEnginesCommand();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 01152dc3d17a71c..adbf3720c05c2de 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -213,6 +213,7 @@ public enum PlanType {
     SHOW_BACKENDS_COMMAND,
     SHOW_BLOCK_RULE_COMMAND,
     SHOW_BROKER_COMMAND,
+    SHOW_CATALOG_COMMAND,
     SHOW_CHARSET_COMMAND,
     SHOW_COLLATION_COMMAND,
     SHOW_CONFIG_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCatalogCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCatalogCommand.java
new file mode 100644
index 000000000000000..32ccee97380a883
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCatalogCommand.java
@@ -0,0 +1,104 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.ShowResultSetMetaData;
+import org.apache.doris.qe.StmtExecutor;
+
+import java.util.List;
+
+/**
+ * Represents the command for show all catalog or desc the specific catalog.
+ */
+public class ShowCatalogCommand extends ShowCommand {
+    private static final ShowResultSetMetaData META_DATA_ALL =
+            ShowResultSetMetaData.builder().addColumn(new Column("CatalogId", ScalarType.BIGINT))
+                .addColumn(new Column("CatalogName", ScalarType.createVarchar(64)))
+                .addColumn(new Column("Type", ScalarType.createStringType()))
+                .addColumn(new Column("IsCurrent", ScalarType.createStringType()))
+                .addColumn(new Column("CreateTime", ScalarType.createStringType()))
+                .addColumn(new Column("LastUpdateTime", ScalarType.createStringType()))
+                .addColumn(new Column("Comment", ScalarType.createStringType()))
+                .build();
+
+    private static final ShowResultSetMetaData META_DATA_SPECIFIC =
+            ShowResultSetMetaData.builder()
+                .addColumn(new Column("Key", ScalarType.createStringType()))
+                .addColumn(new Column("Value", ScalarType.createStringType()))
+                .build();
+
+    private final String catalogName;
+    private final String pattern;
+
+    public ShowCatalogCommand(String catalogName, String pattern) {
+        super(PlanType.SHOW_CATALOG_COMMAND);
+        this.catalogName = catalogName;
+        this.pattern = pattern;
+    }
+
+    @Override
+    public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        List<List<String>> rows = Env.getCurrentEnv().getCatalogMgr()
+                .showCatalogs(catalogName, pattern, ctx.getCurrentCatalog() != null
+                    ? ctx.getCurrentCatalog().getName() : null);
+
+        return new ShowResultSet(getMetaData(), rows);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitShowCatalogCommand(this, context);
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        sb.append("SHOW");
+
+        if (catalogName != null) {
+            sb.append(" CATALOG ");
+            sb.append(catalogName);
+        } else {
+            sb.append(" CATALOGS");
+
+            if (pattern != null) {
+                sb.append(" LIKE ");
+                sb.append("'");
+                sb.append(pattern);
+                sb.append("'");
+            }
+        }
+
+        return sb.toString();
+    }
+
+    public ShowResultSetMetaData getMetaData() {
+        if (catalogName == null) {
+            return META_DATA_ALL;
+        } else {
+            return META_DATA_SPECIFIC;
+        }
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index ed4a1bdb1b3ee2a..2c926c1de1a9a21 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -93,6 +93,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowAuthorsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowBackendsCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowBrokerCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowCatalogCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCharsetCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCollationCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowConfigCommand;
@@ -527,6 +528,10 @@ default R visitShowCreateCatalogCommand(ShowCreateCatalogCommand showCreateCatal
         return visitCommand(showCreateCatalogCommand, context);
     }
 
+    default R visitShowCatalogCommand(ShowCatalogCommand showCatalogCommand, C context) {
+        return visitCommand(showCatalogCommand, context);
+    }
+
     default R visitShowCreateMaterializedViewCommand(ShowCreateMaterializedViewCommand showCreateMtlzViewCommand,
                         C context) {
         return visitCommand(showCreateMtlzViewCommand, context);
diff --git a/regression-test/suites/nereids_p0/show/test_show_catalog.groovy b/regression-test/suites/nereids_p0/show/test_show_catalog.groovy
new file mode 100644
index 000000000000000..cc7300682896684
--- /dev/null
+++ b/regression-test/suites/nereids_p0/show/test_show_catalog.groovy
@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_show_catalog", "query,catalog") {
+
+    String catalog_name = "es"
+
+    sql """drop catalog if exists ${catalog_name}"""
+    sql """create catalog if not exists ${catalog_name} properties (
+            "type"="es",
+            "hosts"="http://127.0.0.1:9200"
+    );"""
+
+    checkNereidsExecute("""show catalog ${catalog_name}""")
+    checkNereidsExecute("""show catalogs like 'e%'""")
+    checkNereidsExecute("""show catalogs """)
+
+    sql """drop catalog if exists ${catalog_name}"""
+}

From 15101f5dfa3a9343a9749279649b62d267e55c5f Mon Sep 17 00:00:00 2001
From: zhiqiang <hezhiqiang@selectdb.com>
Date: Thu, 9 Jan 2025 14:59:13 +0800
Subject: [PATCH 073/140] [fix](profile) Fix profile of broker load (#46676)

---
 .../doris/load/loadv2/BrokerLoadJob.java      |  3 +-
 .../apache/doris/load/loadv2/BulkLoadJob.java |  5 ++++
 .../doris/load/loadv2/LoadLoadingTask.java    |  6 ++--
 .../query_profile/s3_load_profile_test.groovy | 28 ++++++++-----------
 4 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
index 895f46866e0e71e..b2fd6746f2fe692 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
@@ -246,7 +246,8 @@ private void createLoadingTask(Database db, BrokerPendingTaskAttachment attachme
             this.jobProfile = new Profile(
                     true,
                     Integer.valueOf(sessionVariables.getOrDefault(SessionVariable.PROFILE_LEVEL, "3")),
-                    Integer.valueOf(sessionVariables.getOrDefault(SessionVariable.AUTO_PROFILE_THRESHOLD_MS, "500")));
+                    Integer.valueOf(sessionVariables.getOrDefault(SessionVariable.AUTO_PROFILE_THRESHOLD_MS, "-1")));
+            this.jobProfile.getSummaryProfile().setQueryBeginTime();
             // TODO: 怎么给这些 load job 设置 profile 记录时间
             // this.jobProfile.setId("BrokerLoadJob " + id + ". " + label);
         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java
index 3e5742e2b052e18..4f590bd85a2c170 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java
@@ -113,8 +113,13 @@ public BulkLoadJob(EtlJobType jobType, long dbId, String label,
         if (ConnectContext.get() != null) {
             SessionVariable var = ConnectContext.get().getSessionVariable();
             sessionVariables.put(SessionVariable.SQL_MODE, Long.toString(var.getSqlMode()));
+            sessionVariables.put(SessionVariable.AUTO_PROFILE_THRESHOLD_MS,
+                                    Long.toString(var.getAutoProfileThresholdMs()));
+            sessionVariables.put(SessionVariable.PROFILE_LEVEL, Long.toString(var.getProfileLevel()));
         } else {
             sessionVariables.put(SessionVariable.SQL_MODE, String.valueOf(SqlModeHelper.MODE_DEFAULT));
+            sessionVariables.put(SessionVariable.AUTO_PROFILE_THRESHOLD_MS, Long.toString(-1));
+            sessionVariables.put(SessionVariable.PROFILE_LEVEL, Long.toString(3));
         }
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java
index fd64ae7e8ba70de..e6e2c8c07141163 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java
@@ -143,13 +143,13 @@ protected void executeTask() throws Exception {
     }
 
     protected void executeOnce() throws Exception {
-        final boolean enabelProfile = this.jobProfile != null;
+        final boolean enableProfile = this.jobProfile != null;
         // New one query id,
         Coordinator curCoordinator =  EnvFactory.getInstance().createCoordinator(callback.getCallbackId(),
                 loadId, planner.getDescTable(),
                 planner.getFragments(), planner.getScanNodes(), planner.getTimezone(), loadZeroTolerance,
-                enabelProfile);
-        if (enabelProfile) {
+                enableProfile);
+        if (enableProfile) {
             this.jobProfile.addExecutionProfile(curCoordinator.getExecutionProfile());
         }
         curCoordinator.setQueryType(TQueryType.LOAD);
diff --git a/regression-test/suites/query_profile/s3_load_profile_test.groovy b/regression-test/suites/query_profile/s3_load_profile_test.groovy
index be4f38cfdbce7a5..3d1b0390864fa72 100644
--- a/regression-test/suites/query_profile/s3_load_profile_test.groovy
+++ b/regression-test/suites/query_profile/s3_load_profile_test.groovy
@@ -19,7 +19,7 @@ import groovy.json.JsonSlurper
 
 def getProfile = { id ->
         def dst = 'http://' + context.config.feHttpAddress
-        def conn = new URL(dst + "/rest/v1/query_profile/$id").openConnection()
+        def conn = new URL(dst + "/api/profile/text/?query_id=$id").openConnection()
         conn.setRequestMethod("GET")
         def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" + 
                 (context.config.feHttpPassword == null ? "" : context.config.feHttpPassword)).getBytes("UTF-8"))
@@ -31,9 +31,9 @@ def getProfile = { id ->
 suite('s3_load_profile_test') {
     def s3Endpoint = getS3Endpoint()
     def s3Region = getS3Region()
-    sql "drop table if exists dup_tbl_basic;"
+    sql "drop table if exists s3_load_profile_test_dup_tbl_basic;"
     sql """
-    CREATE TABLE dup_tbl_basic
+    CREATE TABLE s3_load_profile_test_dup_tbl_basic
 (
     k00 INT             NOT NULL,
     k01 DATE            NOT NULL,
@@ -100,7 +100,7 @@ PROPERTIES (
 );
 """
     def loadAttribute =new LoadAttributes("s3://${getS3BucketName()}/regression/load/data/basic_data.csv",
-                "dup_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+                "s3_load_profile_test_dup_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
                 "", "", "", "", "")
 
     def ak = getS3AK()
@@ -109,6 +109,7 @@ PROPERTIES (
     sql "set enable_profile=true;"    
     
     def label = "test_s3_load_" + UUID.randomUUID().toString().replace("-", "_")
+    logger.info("s3_load_profile_test_dup_tbl_basic, label: $label")
     loadAttribute.label = label
     def prop = loadAttribute.getPropertiesStr()
 
@@ -167,21 +168,14 @@ PROPERTIES (
             assertTrue(false, "load Timeout: $loadAttribute.label")
         }
     }
-
+    Thread.sleep(5000)
     qt_select """ select count(*) from $loadAttribute.dataDesc.tableName """
-
+    logger.info("jobId: " + jobId)
     def profileString = getProfile(jobId)
-    def profileJson = new JsonSlurper().parseText(profileString)
-    assertEquals(0, profileJson.code)
-    profileDataString = profileJson.data
-    logger.info("profileDataString:" + profileDataString)
-    def taskStateIdx = profileDataString.indexOf("Task&nbsp;&nbsp;State:&nbsp;&nbsp;FINISHED")
-    assertFalse(taskStateIdx == -1)
-    def executionProfileIdx = profileDataString.indexOf("Execution&nbsp;&nbsp;Profile")
-    assertFalse(executionProfileIdx == -1)
-    assertTrue(profileDataString.contains("NumScanners"))
-    assertTrue(profileDataString.contains("RowsProduced"))
-    assertTrue(profileDataString.contains("RowsRead"))
+    logger.info("profileDataString:" + profileString)
+    assertTrue(profileString.contains("NumScanners"))
+    assertTrue(profileString.contains("RowsProduced"))
+    assertTrue(profileString.contains("RowsRead"))
 }
 
 class DataDesc {

From 7d3d36e9b59de7aea8242dbdecd18dab8cc228fa Mon Sep 17 00:00:00 2001
From: TengJianPing <tengjianping@selectdb.com>
Date: Thu, 9 Jan 2025 15:00:37 +0800
Subject: [PATCH 074/140] [improvement](segment) reduce memory usage when open
 segments (#46570)

### What problem does this PR solve?

Issue Number: close #xxx

Related PR: #xxx

Problem Summary:
When there are a lot of segments in one rowset, it will consume plenty
of memory if open all the segments all at once. This PR open segments
one by one and release the `Segment` object immediately if it's not need
to be kept for later use, thus reduce memory footprints dramatically.
---
 be/src/olap/parallel_scanner_builder.cpp      | 18 ++--
 be/src/olap/rowset/beta_rowset.cpp            | 21 ++++-
 be/src/olap/rowset/beta_rowset.h              |  7 +-
 be/src/olap/rowset/beta_rowset_reader.cpp     | 92 +++++--------------
 be/src/olap/rowset/beta_rowset_reader.h       |  8 +-
 be/src/olap/rowset/rowset.cpp                 |  2 -
 be/src/olap/rowset/rowset.h                   |  3 -
 be/src/olap/rowset/rowset_reader.h            |  6 +-
 be/src/olap/rowset/rowset_reader_context.h    |  2 +-
 .../segment_v2/lazy_init_segment_iterator.cpp | 23 ++++-
 .../segment_v2/lazy_init_segment_iterator.h   | 12 ++-
 be/src/olap/segment_loader.cpp                | 56 ++++++-----
 be/src/olap/segment_loader.h                  |  7 ++
 be/test/olap/ordered_data_compaction_test.cpp |  2 -
 be/test/olap/rowid_conversion_test.cpp        |  3 +-
 be/test/olap/segcompaction_mow_test.cpp       |  9 +-
 be/test/olap/segcompaction_test.cpp           | 18 ++--
 be/test/testutil/mock_rowset.h                |  4 -
 be/test/vec/olap/vertical_compaction_test.cpp | 12 ---
 19 files changed, 141 insertions(+), 164 deletions(-)

diff --git a/be/src/olap/parallel_scanner_builder.cpp b/be/src/olap/parallel_scanner_builder.cpp
index 88c69ab5c9a584a..769abe4a9469133 100644
--- a/be/src/olap/parallel_scanner_builder.cpp
+++ b/be/src/olap/parallel_scanner_builder.cpp
@@ -70,7 +70,7 @@ Status ParallelScannerBuilder::_build_scanners_by_rowid(std::list<VScannerSPtr>&
                 continue;
             }
 
-            int segment_start = 0;
+            int64_t segment_start = 0;
             auto split = RowSetSplits(reader->clone());
 
             for (size_t i = 0; i != segments_rows.size(); ++i) {
@@ -171,22 +171,18 @@ Status ParallelScannerBuilder::_load() {
         if (!_state->skip_delete_predicate()) {
             read_source.fill_delete_predicates();
         }
-        bool enable_segment_cache = _state->query_options().__isset.enable_segment_cache
-                                            ? _state->query_options().enable_segment_cache
-                                            : true;
 
         for (auto& rs_split : read_source.rs_splits) {
             auto rowset = rs_split.rs_reader->rowset();
             RETURN_IF_ERROR(rowset->load());
             const auto rowset_id = rowset->rowset_id();
-            SegmentCacheHandle segment_cache_handle;
 
-            RETURN_IF_ERROR(SegmentLoader::instance()->load_segments(
-                    std::dynamic_pointer_cast<BetaRowset>(rowset), &segment_cache_handle,
-                    enable_segment_cache, false));
-
-            for (const auto& segment : segment_cache_handle.get_segments()) {
-                _all_segments_rows[rowset_id].emplace_back(segment->num_rows());
+            auto beta_rowset = std::dynamic_pointer_cast<BetaRowset>(rowset);
+            std::vector<uint32_t> segment_rows;
+            RETURN_IF_ERROR(beta_rowset->get_segment_num_rows(&segment_rows));
+            auto segment_count = rowset->num_segments();
+            for (int64_t i = 0; i != segment_count; i++) {
+                _all_segments_rows[rowset_id].emplace_back(segment_rows[i]);
             }
             _total_rows += rowset->num_rows();
         }
diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp
index a328b1b9e8b90ec..59019f93c6e7ff2 100644
--- a/be/src/olap/rowset/beta_rowset.cpp
+++ b/be/src/olap/rowset/beta_rowset.cpp
@@ -42,6 +42,7 @@
 #include "olap/rowset/segment_v2/inverted_index_cache.h"
 #include "olap/rowset/segment_v2/inverted_index_desc.h"
 #include "olap/rowset/segment_v2/inverted_index_file_reader.h"
+#include "olap/segment_loader.h"
 #include "olap/tablet_schema.h"
 #include "olap/utils.h"
 #include "util/crc32c.h"
@@ -68,9 +69,23 @@ Status BetaRowset::init() {
     return Status::OK(); // no op
 }
 
-Status BetaRowset::do_load(bool /*use_cache*/) {
-    // do nothing.
-    // the segments in this rowset will be loaded by calling load_segments() explicitly.
+Status BetaRowset::get_segment_num_rows(std::vector<uint32_t>* segment_rows) {
+    DCHECK(_rowset_state_machine.rowset_state() == ROWSET_LOADED);
+
+    RETURN_IF_ERROR(_load_segment_rows_once.call([this] {
+        auto segment_count = num_segments();
+        _segments_rows.resize(segment_count);
+        for (int64_t i = 0; i != segment_count; ++i) {
+            SegmentCacheHandle segment_cache_handle;
+            RETURN_IF_ERROR(SegmentLoader::instance()->load_segment(
+                    std::static_pointer_cast<BetaRowset>(shared_from_this()), i,
+                    &segment_cache_handle, false, false));
+            const auto& tmp_segments = segment_cache_handle.get_segments();
+            _segments_rows[i] = tmp_segments[0]->num_rows();
+        }
+        return Status::OK();
+    }));
+    segment_rows->assign(_segments_rows.cbegin(), _segments_rows.cend());
     return Status::OK();
 }
 
diff --git a/be/src/olap/rowset/beta_rowset.h b/be/src/olap/rowset/beta_rowset.h
index 52d5ac5c8a8742f..c9ed568186a8e9a 100644
--- a/be/src/olap/rowset/beta_rowset.h
+++ b/be/src/olap/rowset/beta_rowset.h
@@ -89,6 +89,8 @@ class BetaRowset final : public Rowset {
     Status show_nested_index_file(rapidjson::Value* rowset_value,
                                   rapidjson::Document::AllocatorType& allocator);
 
+    Status get_segment_num_rows(std::vector<uint32_t>* segment_rows);
+
 protected:
     BetaRowset(const TabletSchemaSPtr& schema, const RowsetMetaSharedPtr& rowset_meta,
                std::string tablet_path);
@@ -96,8 +98,6 @@ class BetaRowset final : public Rowset {
     // init segment groups
     Status init() override;
 
-    Status do_load(bool use_cache) override;
-
     void do_close() override;
 
     Status check_current_rowset_segment() override;
@@ -107,6 +107,9 @@ class BetaRowset final : public Rowset {
 private:
     friend class RowsetFactory;
     friend class BetaRowsetReader;
+
+    DorisCallOnce<Status> _load_segment_rows_once;
+    std::vector<uint32_t> _segments_rows;
 };
 
 } // namespace doris
diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp b/be/src/olap/rowset/beta_rowset_reader.cpp
index 9a4d71587a02c10..822916819fed134 100644
--- a/be/src/olap/rowset/beta_rowset_reader.cpp
+++ b/be/src/olap/rowset/beta_rowset_reader.cpp
@@ -215,7 +215,6 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context
         _read_options.io_ctx.expiration_time = 0;
     }
 
-    // load segments
     bool enable_segment_cache = true;
     auto* state = read_context->runtime_state;
     if (state != nullptr) {
@@ -226,76 +225,41 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context
     // When reader type is for query, session variable `enable_segment_cache` should be respected.
     bool should_use_cache = use_cache || (_read_context->reader_type == ReaderType::READER_QUERY &&
                                           enable_segment_cache);
-    SegmentCacheHandle segment_cache_handle;
-    {
-        SCOPED_RAW_TIMER(&_stats->rowset_reader_load_segments_timer_ns);
-        RETURN_IF_ERROR(SegmentLoader::instance()->load_segments(
-                _rowset, &segment_cache_handle, should_use_cache,
-                /*need_load_pk_index_and_bf*/ false));
-    }
-
-    // create iterator for each segment
-    auto& segments = segment_cache_handle.get_segments();
-    _segments_rows.resize(segments.size());
-    for (size_t i = 0; i < segments.size(); i++) {
-        _segments_rows[i] = segments[i]->num_rows();
-    }
-    if (_read_context->record_rowids) {
-        // init segment rowid map for rowid conversion
-        std::vector<uint32_t> segment_num_rows;
-        RETURN_IF_ERROR(get_segment_num_rows(&segment_num_rows));
-        RETURN_IF_ERROR(_read_context->rowid_conversion->init_segment_map(rowset()->rowset_id(),
-                                                                          segment_num_rows));
-    }
 
+    auto segment_count = _rowset->num_segments();
     auto [seg_start, seg_end] = _segment_offsets;
+    // If seg_start == seg_end, it means that the segments of a rowset is not
+    // split scanned by multiple scanners, and the rowset reader is used to read the whole rowset.
     if (seg_start == seg_end) {
         seg_start = 0;
-        seg_end = segments.size();
+        seg_end = segment_count;
+    }
+    if (_read_context->record_rowids && _read_context->rowid_conversion) {
+        // init segment rowid map for rowid conversion
+        std::vector<uint32_t> segment_rows;
+        RETURN_IF_ERROR(_rowset->get_segment_num_rows(&segment_rows));
+        RETURN_IF_ERROR(_read_context->rowid_conversion->init_segment_map(rowset()->rowset_id(),
+                                                                          segment_rows));
     }
 
-    const bool is_merge_iterator = _is_merge_iterator();
-    const bool use_lazy_init_iterators =
-            !is_merge_iterator && _read_context->reader_type == ReaderType::READER_QUERY;
-    for (int i = seg_start; i < seg_end; i++) {
+    for (int64_t i = seg_start; i < seg_end; i++) {
         SCOPED_RAW_TIMER(&_stats->rowset_reader_create_iterators_timer_ns);
-        auto& seg_ptr = segments[i];
         std::unique_ptr<RowwiseIterator> iter;
 
-        if (use_lazy_init_iterators) {
-            /// For non-merging iterators, we don't need to initialize them all at once when creating them.
-            /// Instead, we should initialize each iterator separately when really using them.
-            /// This optimization minimizes the lifecycle of resources like column readers
-            /// and prevents excessive memory consumption, especially for wide tables.
-            if (_segment_row_ranges.empty()) {
-                _read_options.row_ranges.clear();
-                iter = std::make_unique<LazyInitSegmentIterator>(seg_ptr, _input_schema,
-                                                                 _read_options);
-            } else {
-                DCHECK_EQ(seg_end - seg_start, _segment_row_ranges.size());
-                auto local_options = _read_options;
-                local_options.row_ranges = _segment_row_ranges[i - seg_start];
-                iter = std::make_unique<LazyInitSegmentIterator>(seg_ptr, _input_schema,
-                                                                 local_options);
-            }
+        /// For iterators, we don't need to initialize them all at once when creating them.
+        /// Instead, we should initialize each iterator separately when really using them.
+        /// This optimization minimizes the lifecycle of resources like column readers
+        /// and prevents excessive memory consumption, especially for wide tables.
+        if (_segment_row_ranges.empty()) {
+            _read_options.row_ranges.clear();
+            iter = std::make_unique<LazyInitSegmentIterator>(_rowset, i, should_use_cache,
+                                                             _input_schema, _read_options);
         } else {
-            Status status;
-            /// If `_segment_row_ranges` is empty, the segment is not split.
-            if (_segment_row_ranges.empty()) {
-                _read_options.row_ranges.clear();
-                status = seg_ptr->new_iterator(_input_schema, _read_options, &iter);
-            } else {
-                DCHECK_EQ(seg_end - seg_start, _segment_row_ranges.size());
-                auto local_options = _read_options;
-                local_options.row_ranges = _segment_row_ranges[i - seg_start];
-                status = seg_ptr->new_iterator(_input_schema, local_options, &iter);
-            }
-
-            if (!status.ok()) {
-                LOG(WARNING) << "failed to create iterator[" << seg_ptr->id()
-                             << "]: " << status.to_string();
-                return Status::Error<ROWSET_READER_INIT>(status.to_string());
-            }
+            DCHECK_EQ(seg_end - seg_start, _segment_row_ranges.size());
+            auto local_options = _read_options;
+            local_options.row_ranges = _segment_row_ranges[i - seg_start];
+            iter = std::make_unique<LazyInitSegmentIterator>(_rowset, i, should_use_cache,
+                                                             _input_schema, local_options);
         }
 
         if (iter->empty()) {
@@ -423,10 +387,4 @@ bool BetaRowsetReader::_should_push_down_value_predicates() const {
              _read_context->sequence_id_idx == -1) ||
             _read_context->enable_unique_key_merge_on_write);
 }
-
-Status BetaRowsetReader::get_segment_num_rows(std::vector<uint32_t>* segment_num_rows) {
-    segment_num_rows->assign(_segments_rows.cbegin(), _segments_rows.cend());
-    return Status::OK();
-}
-
 } // namespace doris
diff --git a/be/src/olap/rowset/beta_rowset_reader.h b/be/src/olap/rowset/beta_rowset_reader.h
index 33b2fb6a58c08ba..b191480f7c7d049 100644
--- a/be/src/olap/rowset/beta_rowset_reader.h
+++ b/be/src/olap/rowset/beta_rowset_reader.h
@@ -80,8 +80,6 @@ class BetaRowsetReader : public RowsetReader {
         return _iterator->current_block_row_locations(locations);
     }
 
-    Status get_segment_num_rows(std::vector<uint32_t>* segment_num_rows) override;
-
     bool update_profile(RuntimeProfile* profile) override;
 
     RowsetReaderSharedPtr clone() override;
@@ -97,7 +95,7 @@ class BetaRowsetReader : public RowsetReader {
                _rowset->rowset_meta()->is_segments_overlapping() && _get_segment_num() > 1;
     }
 
-    int32_t _get_segment_num() const {
+    int64_t _get_segment_num() const {
         auto [seg_start, seg_end] = _segment_offsets;
         if (seg_start == seg_end) {
             seg_start = 0;
@@ -108,7 +106,7 @@ class BetaRowsetReader : public RowsetReader {
 
     DorisCallOnce<Status> _init_iter_once;
 
-    std::pair<int, int> _segment_offsets;
+    std::pair<int64_t, int64_t> _segment_offsets;
     std::vector<RowRanges> _segment_row_ranges;
 
     SchemaSPtr _input_schema;
@@ -120,8 +118,6 @@ class BetaRowsetReader : public RowsetReader {
 
     std::unique_ptr<RowwiseIterator> _iterator;
 
-    std::vector<uint32_t> _segments_rows;
-
     StorageReadOptions _read_options;
 
     bool _empty = false;
diff --git a/be/src/olap/rowset/rowset.cpp b/be/src/olap/rowset/rowset.cpp
index ac3a2a7a1dc5c2a..0fd8e60f7cefdf9 100644
--- a/be/src/olap/rowset/rowset.cpp
+++ b/be/src/olap/rowset/rowset.cpp
@@ -67,8 +67,6 @@ Status Rowset::load(bool use_cache) {
         std::lock_guard load_lock(_lock);
         // after lock, if rowset state is ROWSET_UNLOADING, it is ok to return
         if (_rowset_state_machine.rowset_state() == ROWSET_UNLOADED) {
-            // first do load, then change the state
-            RETURN_IF_ERROR(do_load(use_cache));
             RETURN_IF_ERROR(_rowset_state_machine.on_load());
         }
     }
diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h
index 98d88ba19f2068d..047a61a39ea22ea 100644
--- a/be/src/olap/rowset/rowset.h
+++ b/be/src/olap/rowset/rowset.h
@@ -321,9 +321,6 @@ class Rowset : public std::enable_shared_from_this<Rowset>, public MetadataAdder
     // this is non-public because all clients should use RowsetFactory to obtain pointer to initialized Rowset
     virtual Status init() = 0;
 
-    // The actual implementation of load(). Guaranteed by to called exactly once.
-    virtual Status do_load(bool use_cache) = 0;
-
     // release resources in this api
     virtual void do_close() = 0;
 
diff --git a/be/src/olap/rowset/rowset_reader.h b/be/src/olap/rowset/rowset_reader.h
index 58c0f592b9c5451..6c637f47cc17e68 100644
--- a/be/src/olap/rowset/rowset_reader.h
+++ b/be/src/olap/rowset/rowset_reader.h
@@ -40,7 +40,7 @@ struct RowSetSplits {
     // if segment_offsets is not empty, means we only scan
     // [pair.first, pair.second) segment in rs_reader, only effective in dup key
     // and pipeline
-    std::pair<int, int> segment_offsets;
+    std::pair<int64_t, int64_t> segment_offsets;
 
     // RowRanges of each segment.
     std::vector<RowRanges> segment_row_ranges;
@@ -83,10 +83,6 @@ class RowsetReader {
         return Status::NotSupported("to be implemented");
     }
 
-    virtual Status get_segment_num_rows(std::vector<uint32_t>* segment_num_rows) {
-        return Status::NotSupported("to be implemented");
-    }
-
     virtual bool update_profile(RuntimeProfile* profile) = 0;
 
     virtual RowsetReaderSharedPtr clone() = 0;
diff --git a/be/src/olap/rowset/rowset_reader_context.h b/be/src/olap/rowset/rowset_reader_context.h
index fd4fe7a18234f1a..2dd7132890247d9 100644
--- a/be/src/olap/rowset/rowset_reader_context.h
+++ b/be/src/olap/rowset/rowset_reader_context.h
@@ -76,7 +76,7 @@ struct RowsetReaderContext {
     bool enable_unique_key_merge_on_write = false;
     const DeleteBitmap* delete_bitmap = nullptr;
     bool record_rowids = false;
-    RowIdConversion* rowid_conversion;
+    RowIdConversion* rowid_conversion = nullptr;
     bool is_key_column_group = false;
     const std::set<int32_t>* output_columns = nullptr;
     RowsetId rowset_id;
diff --git a/be/src/olap/rowset/segment_v2/lazy_init_segment_iterator.cpp b/be/src/olap/rowset/segment_v2/lazy_init_segment_iterator.cpp
index d70df5a7baeae9c..77e2310fc48cc13 100644
--- a/be/src/olap/rowset/segment_v2/lazy_init_segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/lazy_init_segment_iterator.cpp
@@ -17,11 +17,18 @@
 
 #include "olap/rowset/segment_v2/lazy_init_segment_iterator.h"
 
+#include "olap/segment_loader.h"
+
 namespace doris::segment_v2 {
 
-LazyInitSegmentIterator::LazyInitSegmentIterator(std::shared_ptr<Segment> segment,
-                                                 SchemaSPtr schema, const StorageReadOptions& opts)
-        : _schema(std::move(schema)), _segment(std::move(segment)), _read_options(opts) {}
+LazyInitSegmentIterator::LazyInitSegmentIterator(BetaRowsetSharedPtr rowset, int64_t segment_id,
+                                                 bool should_use_cache, SchemaSPtr schema,
+                                                 const StorageReadOptions& opts)
+        : _rowset(std::move(rowset)),
+          _segment_id(segment_id),
+          _should_use_cache(should_use_cache),
+          _schema(std::move(schema)),
+          _read_options(opts) {}
 
 /// Here do not use the argument of `opts`,
 /// see where the iterator is created in `BetaRowsetReader::get_segment_iterators`
@@ -31,7 +38,15 @@ Status LazyInitSegmentIterator::init(const StorageReadOptions& /*opts*/) {
         return Status::OK();
     }
 
-    RETURN_IF_ERROR(_segment->new_iterator(_schema, _read_options, &_inner_iterator));
+    std::shared_ptr<Segment> segment;
+    {
+        SegmentCacheHandle segment_cache_handle;
+        RETURN_IF_ERROR(SegmentLoader::instance()->load_segment(
+                _rowset, _segment_id, &segment_cache_handle, _should_use_cache, false));
+        const auto& tmp_segments = segment_cache_handle.get_segments();
+        segment = tmp_segments[0];
+    }
+    RETURN_IF_ERROR(segment->new_iterator(_schema, _read_options, &_inner_iterator));
     return _inner_iterator->init(_read_options);
 }
 
diff --git a/be/src/olap/rowset/segment_v2/lazy_init_segment_iterator.h b/be/src/olap/rowset/segment_v2/lazy_init_segment_iterator.h
index 923c540c4569999..c31918d092c0d17 100644
--- a/be/src/olap/rowset/segment_v2/lazy_init_segment_iterator.h
+++ b/be/src/olap/rowset/segment_v2/lazy_init_segment_iterator.h
@@ -22,14 +22,18 @@
 #include "olap/rowset/segment_v2/segment_iterator.h"
 #include "vec/core/block.h"
 
+namespace doris {
+class BetaRowset;
+using BetaRowsetSharedPtr = std::shared_ptr<BetaRowset>;
+}; // namespace doris
 namespace doris::segment_v2 {
 
 using namespace vectorized;
 
 class LazyInitSegmentIterator : public RowwiseIterator {
 public:
-    LazyInitSegmentIterator(std::shared_ptr<Segment> segment, SchemaSPtr schema,
-                            const StorageReadOptions& opts);
+    LazyInitSegmentIterator(BetaRowsetSharedPtr rowset, int64_t segment_id, bool should_use_cache,
+                            SchemaSPtr schema, const StorageReadOptions& opts);
 
     ~LazyInitSegmentIterator() override = default;
 
@@ -59,8 +63,10 @@ class LazyInitSegmentIterator : public RowwiseIterator {
 
 private:
     bool _need_lazy_init {true};
+    BetaRowsetSharedPtr _rowset;
+    int64_t _segment_id {-1};
+    bool _should_use_cache {false};
     SchemaSPtr _schema = nullptr;
-    std::shared_ptr<Segment> _segment;
     StorageReadOptions _read_options;
     RowwiseIteratorUPtr _inner_iterator;
 };
diff --git a/be/src/olap/segment_loader.cpp b/be/src/olap/segment_loader.cpp
index 4240f7e250a06b0..1bd21ad4e553c0a 100644
--- a/be/src/olap/segment_loader.cpp
+++ b/be/src/olap/segment_loader.cpp
@@ -52,6 +52,37 @@ void SegmentCache::erase(const SegmentCache::CacheKey& key) {
     LRUCachePolicy::erase(key.encode());
 }
 
+Status SegmentLoader::load_segment(const BetaRowsetSharedPtr& rowset, int64_t segment_id,
+                                   SegmentCacheHandle* cache_handle, bool use_cache,
+                                   bool need_load_pk_index_and_bf,
+                                   OlapReaderStatistics* index_load_stats) {
+    SegmentCache::CacheKey cache_key(rowset->rowset_id(), segment_id);
+    if (_segment_cache->lookup(cache_key, cache_handle)) {
+        // Has to check the segment status here, because the segment in cache may has something wrong during
+        // load index or create column reader.
+        // Not merge this if logic with previous to make the logic more clear.
+        if (cache_handle->pop_unhealthy_segment() == nullptr) {
+            return Status::OK();
+        }
+    }
+    // If the segment is not healthy, then will create a new segment and will replace the unhealthy one in SegmentCache.
+    segment_v2::SegmentSharedPtr segment;
+    RETURN_IF_ERROR(rowset->load_segment(segment_id, &segment));
+    if (need_load_pk_index_and_bf) {
+        RETURN_IF_ERROR(segment->load_pk_index_and_bf(index_load_stats));
+    }
+    if (use_cache && !config::disable_segment_cache) {
+        // memory of SegmentCache::CacheValue will be handled by SegmentCache
+        auto* cache_value = new SegmentCache::CacheValue(segment);
+        _cache_mem_usage += segment->meta_mem_usage();
+        _segment_cache->insert(cache_key, *cache_value, cache_handle);
+    } else {
+        cache_handle->push_segment(std::move(segment));
+    }
+
+    return Status::OK();
+}
+
 Status SegmentLoader::load_segments(const BetaRowsetSharedPtr& rowset,
                                     SegmentCacheHandle* cache_handle, bool use_cache,
                                     bool need_load_pk_index_and_bf,
@@ -60,29 +91,8 @@ Status SegmentLoader::load_segments(const BetaRowsetSharedPtr& rowset,
         return Status::OK();
     }
     for (int64_t i = 0; i < rowset->num_segments(); i++) {
-        SegmentCache::CacheKey cache_key(rowset->rowset_id(), i);
-        if (_segment_cache->lookup(cache_key, cache_handle)) {
-            // Has to check the segment status here, because the segment in cache may has something wrong during
-            // load index or create column reader.
-            // Not merge this if logic with previous to make the logic more clear.
-            if (cache_handle->pop_unhealthy_segment() == nullptr) {
-                continue;
-            }
-        }
-        // If the segment is not healthy, then will create a new segment and will replace the unhealthy one in SegmentCache.
-        segment_v2::SegmentSharedPtr segment;
-        RETURN_IF_ERROR(rowset->load_segment(i, &segment));
-        if (need_load_pk_index_and_bf) {
-            RETURN_IF_ERROR(segment->load_pk_index_and_bf(index_load_stats));
-        }
-        if (use_cache && !config::disable_segment_cache) {
-            // memory of SegmentCache::CacheValue will be handled by SegmentCache
-            auto* cache_value = new SegmentCache::CacheValue(segment);
-            _cache_mem_usage += segment->meta_mem_usage();
-            _segment_cache->insert(cache_key, *cache_value, cache_handle);
-        } else {
-            cache_handle->push_segment(std::move(segment));
-        }
+        RETURN_IF_ERROR(load_segment(rowset, i, cache_handle, use_cache, need_load_pk_index_and_bf,
+                                     index_load_stats));
     }
     cache_handle->set_inited();
     return Status::OK();
diff --git a/be/src/olap/segment_loader.h b/be/src/olap/segment_loader.h
index 2c5b1ed200dde78..d1813f434cb2789 100644
--- a/be/src/olap/segment_loader.h
+++ b/be/src/olap/segment_loader.h
@@ -120,6 +120,13 @@ class SegmentLoader {
                          bool use_cache = false, bool need_load_pk_index_and_bf = false,
                          OlapReaderStatistics* index_load_stats = nullptr);
 
+    // Load one segment of "rowset", return the "cache_handle" which contains segments.
+    // If use_cache is true, it will be loaded from _cache.
+    Status load_segment(const BetaRowsetSharedPtr& rowset, int64_t segment_id,
+                        SegmentCacheHandle* cache_handle, bool use_cache = false,
+                        bool need_load_pk_index_and_bf = false,
+                        OlapReaderStatistics* index_load_stats = nullptr);
+
     void erase_segment(const SegmentCache::CacheKey& key);
 
     void erase_segments(const RowsetId& rowset_id, int64_t num_segments);
diff --git a/be/test/olap/ordered_data_compaction_test.cpp b/be/test/olap/ordered_data_compaction_test.cpp
index 058ed52dd995dc7..831e3031378e4e5 100644
--- a/be/test/olap/ordered_data_compaction_test.cpp
+++ b/be/test/olap/ordered_data_compaction_test.cpp
@@ -464,8 +464,6 @@ TEST_F(OrderedDataCompactionTest, test_01) {
     EXPECT_EQ(Status::Error<END_OF_FILE>(""), s);
     EXPECT_EQ(out_rowset->rowset_meta()->num_rows(), output_data.size());
     EXPECT_EQ(output_data.size(), num_input_rowset * num_segments * rows_per_segment);
-    std::vector<uint32_t> segment_num_rows;
-    EXPECT_TRUE(output_rs_reader->get_segment_num_rows(&segment_num_rows).ok());
     // check vertical compaction result
     for (auto id = 0; id < output_data.size(); id++) {
         LOG(INFO) << "output data: " << std::get<0>(output_data[id]) << " "
diff --git a/be/test/olap/rowid_conversion_test.cpp b/be/test/olap/rowid_conversion_test.cpp
index df56cd0559e4d87..52d81f2100aba46 100644
--- a/be/test/olap/rowid_conversion_test.cpp
+++ b/be/test/olap/rowid_conversion_test.cpp
@@ -382,8 +382,9 @@ class TestRowIdConversion : public testing::TestWithParam<std::tuple<KeysType, b
         } while (s.ok());
         EXPECT_TRUE(s.is<END_OF_FILE>()) << s;
         EXPECT_EQ(out_rowset->rowset_meta()->num_rows(), output_data.size());
+        auto beta_rowset = std::dynamic_pointer_cast<BetaRowset>(out_rowset);
         std::vector<uint32_t> segment_num_rows;
-        EXPECT_TRUE(output_rs_reader->get_segment_num_rows(&segment_num_rows).ok());
+        EXPECT_TRUE(beta_rowset->get_segment_num_rows(&segment_num_rows).ok());
         if (has_delete_handler) {
             // All keys less than 1000 are deleted by delete handler
             for (auto& item : output_data) {
diff --git a/be/test/olap/segcompaction_mow_test.cpp b/be/test/olap/segcompaction_mow_test.cpp
index 62a3232889dedee..5463de03f2bf194 100644
--- a/be/test/olap/segcompaction_mow_test.cpp
+++ b/be/test/olap/segcompaction_mow_test.cpp
@@ -239,7 +239,6 @@ class SegCompactionMoWTest : public ::testing::TestWithParam<std::string> {
         reader_context.stats = &_stats;
         reader_context.delete_bitmap = delete_bitmap.get();
 
-        std::vector<uint32_t> segment_num_rows;
         Status s;
 
         // without predicates
@@ -280,7 +279,9 @@ class SegCompactionMoWTest : public ::testing::TestWithParam<std::string> {
             EXPECT_EQ(Status::Error<END_OF_FILE>(""), s);
             EXPECT_EQ(rowset->rowset_meta()->num_rows(), expect_total_rows);
             EXPECT_EQ(num_rows_read, expect_total_rows - rows_mark_deleted);
-            EXPECT_TRUE(rowset_reader->get_segment_num_rows(&segment_num_rows).ok());
+            auto beta_rowset = std::dynamic_pointer_cast<BetaRowset>(rowset);
+            std::vector<uint32_t> segment_num_rows;
+            EXPECT_TRUE(beta_rowset->get_segment_num_rows(&segment_num_rows).ok());
             size_t total_num_rows = 0;
             for (const auto& i : segment_num_rows) {
                 total_num_rows += i;
@@ -307,7 +308,6 @@ TEST_P(SegCompactionMoWTest, SegCompactionThenRead) {
     config::segcompaction_candidate_max_rows = 6000; // set threshold above
                                                      // rows_per_segment
     config::segcompaction_batch_size = 10;
-    std::vector<uint32_t> segment_num_rows;
     DeleteBitmapPtr delete_bitmap = std::make_shared<DeleteBitmap>(TABLET_ID);
     uint32_t rows_mark_deleted = 0;
     { // write `num_segments * rows_per_segment` rows to rowset
@@ -413,7 +413,6 @@ TEST_F(SegCompactionMoWTest, SegCompactionInterleaveWithBig_ooooOOoOooooooooO) {
     DeleteBitmapPtr delete_bitmap = std::make_shared<DeleteBitmap>(TABLET_ID);
     uint32_t rows_mark_deleted = 0;
     uint32_t total_written_rows = 0;
-    std::vector<uint32_t> segment_num_rows;
     { // write `num_segments * rows_per_segment` rows to rowset
         RowsetWriterContext writer_context;
         create_rowset_writer_context(20048, tablet_schema, &writer_context);
@@ -641,7 +640,6 @@ TEST_F(SegCompactionMoWTest, SegCompactionInterleaveWithBig_OoOoO) {
     RowsetSharedPtr rowset;
     config::segcompaction_candidate_max_rows = 6000; // set threshold above
     config::segcompaction_batch_size = 5;
-    std::vector<uint32_t> segment_num_rows;
     DeleteBitmapPtr delete_bitmap = std::make_shared<DeleteBitmap>(TABLET_ID);
     uint32_t rows_mark_deleted = 0;
     uint32_t total_written_rows = 0;
@@ -832,7 +830,6 @@ TEST_F(SegCompactionMoWTest, SegCompactionNotTrigger) {
     config::segcompaction_candidate_max_rows = 6000; // set threshold above
                                                      // rows_per_segment
     config::segcompaction_batch_size = 10;
-    std::vector<uint32_t> segment_num_rows;
     DeleteBitmapPtr delete_bitmap = std::make_shared<DeleteBitmap>(TABLET_ID);
     uint32_t rows_mark_deleted = 0;
     { // write `num_segments * rows_per_segment` rows to rowset
diff --git a/be/test/olap/segcompaction_test.cpp b/be/test/olap/segcompaction_test.cpp
index 4029467dd42fbb8..4a1dfc63fd9b35d 100644
--- a/be/test/olap/segcompaction_test.cpp
+++ b/be/test/olap/segcompaction_test.cpp
@@ -291,7 +291,6 @@ TEST_F(SegCompactionTest, SegCompactionThenRead) {
     config::segcompaction_candidate_max_rows = 6000; // set threshold above
                                                      // rows_per_segment
     config::segcompaction_batch_size = 10;
-    std::vector<uint32_t> segment_num_rows;
     { // write `num_segments * rows_per_segment` rows to rowset
         RowsetWriterContext writer_context;
         create_rowset_writer_context(10047, tablet_schema, &writer_context);
@@ -387,7 +386,9 @@ TEST_F(SegCompactionTest, SegCompactionThenRead) {
             EXPECT_EQ(Status::Error<END_OF_FILE>(""), s);
             EXPECT_EQ(rowset->rowset_meta()->num_rows(), num_rows_read);
             EXPECT_EQ(num_rows_read, num_segments * rows_per_segment);
-            EXPECT_TRUE(rowset_reader->get_segment_num_rows(&segment_num_rows).ok());
+            auto beta_rowset = std::dynamic_pointer_cast<BetaRowset>(rowset);
+            std::vector<uint32_t> segment_num_rows;
+            EXPECT_TRUE(beta_rowset->get_segment_num_rows(&segment_num_rows).ok());
             size_t total_num_rows = 0;
             for (const auto& i : segment_num_rows) {
                 total_num_rows += i;
@@ -406,7 +407,6 @@ TEST_F(SegCompactionTest, SegCompactionInterleaveWithBig_ooooOOoOooooooooO) {
     RowsetSharedPtr rowset;
     config::segcompaction_candidate_max_rows = 6000; // set threshold above
                                                      // rows_per_segment
-    std::vector<uint32_t> segment_num_rows;
     { // write `num_segments * rows_per_segment` rows to rowset
         RowsetWriterContext writer_context;
         create_rowset_writer_context(10048, tablet_schema, &writer_context);
@@ -561,7 +561,6 @@ TEST_F(SegCompactionTest, SegCompactionInterleaveWithBig_OoOoO) {
     RowsetSharedPtr rowset;
     config::segcompaction_candidate_max_rows = 6000; // set threshold above
     config::segcompaction_batch_size = 5;
-    std::vector<uint32_t> segment_num_rows;
     { // write `num_segments * rows_per_segment` rows to rowset
         RowsetWriterContext writer_context;
         create_rowset_writer_context(10049, tablet_schema, &writer_context);
@@ -693,7 +692,6 @@ TEST_F(SegCompactionTest, SegCompactionThenReadUniqueTableSmall) {
     config::segcompaction_candidate_max_rows = 6000; // set threshold above
                                                      // rows_per_segment
     config::segcompaction_batch_size = 3;
-    std::vector<uint32_t> segment_num_rows;
     { // write `num_segments * rows_per_segment` rows to rowset
         RowsetWriterContext writer_context;
         create_rowset_writer_context(10051, tablet_schema, &writer_context);
@@ -894,7 +892,9 @@ TEST_F(SegCompactionTest, SegCompactionThenReadUniqueTableSmall) {
             // duplicated keys between segments are counted duplicately
             // so actual read by rowset reader is less or equal to it
             EXPECT_GE(rowset->rowset_meta()->num_rows(), num_rows_read);
-            EXPECT_TRUE(rowset_reader->get_segment_num_rows(&segment_num_rows).ok());
+            auto beta_rowset = std::dynamic_pointer_cast<BetaRowset>(rowset);
+            std::vector<uint32_t> segment_num_rows;
+            EXPECT_TRUE(beta_rowset->get_segment_num_rows(&segment_num_rows).ok());
             size_t total_num_rows = 0;
             for (const auto& i : segment_num_rows) {
                 total_num_rows += i;
@@ -926,7 +926,6 @@ TEST_F(SegCompactionTest, CreateSegCompactionWriter) {
     config::segcompaction_candidate_max_rows = 6000; // set threshold above
     // rows_per_segment
     config::segcompaction_batch_size = 3;
-    std::vector<uint32_t> segment_num_rows;
     {
         RowsetWriterContext writer_context;
         create_rowset_writer_context(10052, tablet_schema, &writer_context);
@@ -956,7 +955,6 @@ TEST_F(SegCompactionTest, SegCompactionThenReadAggTableSmall) {
     config::segcompaction_candidate_max_rows = 6000; // set threshold above
                                                      // rows_per_segment
     config::segcompaction_batch_size = 3;
-    std::vector<uint32_t> segment_num_rows;
     { // write `num_segments * rows_per_segment` rows to rowset
         RowsetWriterContext writer_context;
         create_rowset_writer_context(10052, tablet_schema, &writer_context);
@@ -1159,7 +1157,9 @@ TEST_F(SegCompactionTest, SegCompactionThenReadAggTableSmall) {
             // duplicated keys between segments are counted duplicately
             // so actual read by rowset reader is less or equal to it
             EXPECT_GE(rowset->rowset_meta()->num_rows(), num_rows_read);
-            EXPECT_TRUE(rowset_reader->get_segment_num_rows(&segment_num_rows).ok());
+            auto beta_rowset = std::dynamic_pointer_cast<BetaRowset>(rowset);
+            std::vector<uint32_t> segment_num_rows;
+            EXPECT_TRUE(beta_rowset->get_segment_num_rows(&segment_num_rows).ok());
             size_t total_num_rows = 0;
             for (const auto& i : segment_num_rows) {
                 total_num_rows += i;
diff --git a/be/test/testutil/mock_rowset.h b/be/test/testutil/mock_rowset.h
index 36ef64fd9ec9fb9..f1472b435b5ac2a 100644
--- a/be/test/testutil/mock_rowset.h
+++ b/be/test/testutil/mock_rowset.h
@@ -73,10 +73,6 @@ class MockRowset : public Rowset {
 
     Status init() override { return Status::NotSupported("MockRowset not support this method."); }
 
-    Status do_load(bool use_cache) override {
-        return Status::NotSupported("MockRowset not support this method.");
-    }
-
     void do_close() override {
         // Do nothing.
     }
diff --git a/be/test/vec/olap/vertical_compaction_test.cpp b/be/test/vec/olap/vertical_compaction_test.cpp
index 4c4409a75068c1f..dd6f6932efcb266 100644
--- a/be/test/vec/olap/vertical_compaction_test.cpp
+++ b/be/test/vec/olap/vertical_compaction_test.cpp
@@ -521,8 +521,6 @@ TEST_F(VerticalCompactionTest, TestDupKeyVerticalMerge) {
     EXPECT_EQ(Status::Error<END_OF_FILE>(""), s);
     EXPECT_EQ(out_rowset->rowset_meta()->num_rows(), output_data.size());
     EXPECT_EQ(output_data.size(), num_input_rowset * num_segments * rows_per_segment);
-    std::vector<uint32_t> segment_num_rows;
-    EXPECT_TRUE(output_rs_reader->get_segment_num_rows(&segment_num_rows).ok());
     // check vertical compaction result
     for (auto id = 0; id < output_data.size(); id++) {
         LOG(INFO) << "output data: " << std::get<0>(output_data[id]) << " "
@@ -628,8 +626,6 @@ TEST_F(VerticalCompactionTest, TestDupWithoutKeyVerticalMerge) {
     EXPECT_EQ(Status::Error<END_OF_FILE>(""), s);
     EXPECT_EQ(out_rowset->rowset_meta()->num_rows(), output_data.size());
     EXPECT_EQ(output_data.size(), num_input_rowset * num_segments * rows_per_segment);
-    std::vector<uint32_t> segment_num_rows;
-    EXPECT_TRUE(output_rs_reader->get_segment_num_rows(&segment_num_rows).ok());
     // check vertical compaction result
     for (auto id = 0; id < output_data.size(); id++) {
         LOG(INFO) << "output data: " << std::get<0>(output_data[id]) << " "
@@ -736,8 +732,6 @@ TEST_F(VerticalCompactionTest, TestUniqueKeyVerticalMerge) {
     EXPECT_EQ(Status::Error<END_OF_FILE>(""), s);
     EXPECT_EQ(out_rowset->rowset_meta()->num_rows(), output_data.size());
     EXPECT_EQ(output_data.size(), num_segments * rows_per_segment);
-    std::vector<uint32_t> segment_num_rows;
-    EXPECT_TRUE(output_rs_reader->get_segment_num_rows(&segment_num_rows).ok());
     // check vertical compaction result
     for (auto id = 0; id < output_data.size(); id++) {
         LOG(INFO) << "output data: " << std::get<0>(output_data[id]) << " "
@@ -848,8 +842,6 @@ TEST_F(VerticalCompactionTest, TestDupKeyVerticalMergeWithDelete) {
     EXPECT_EQ(out_rowset->rowset_meta()->num_rows(), output_data.size());
     EXPECT_EQ(output_data.size(),
               num_input_rowset * num_segments * rows_per_segment - num_input_rowset * 100);
-    std::vector<uint32_t> segment_num_rows;
-    EXPECT_TRUE(output_rs_reader->get_segment_num_rows(&segment_num_rows).ok());
     // All keys less than 1000 are deleted by delete handler
     for (auto& item : output_data) {
         ASSERT_GE(std::get<0>(item), 100);
@@ -951,8 +943,6 @@ TEST_F(VerticalCompactionTest, TestDupWithoutKeyVerticalMergeWithDelete) {
     EXPECT_EQ(out_rowset->rowset_meta()->num_rows(), output_data.size());
     EXPECT_EQ(output_data.size(),
               num_input_rowset * num_segments * rows_per_segment - num_input_rowset * 100);
-    std::vector<uint32_t> segment_num_rows;
-    EXPECT_TRUE(output_rs_reader->get_segment_num_rows(&segment_num_rows).ok());
     // All keys less than 1000 are deleted by delete handler
     for (auto& item : output_data) {
         ASSERT_GE(std::get<0>(item), 100);
@@ -1042,8 +1032,6 @@ TEST_F(VerticalCompactionTest, TestAggKeyVerticalMerge) {
     EXPECT_EQ(Status::Error<END_OF_FILE>(""), s);
     EXPECT_EQ(out_rowset->rowset_meta()->num_rows(), output_data.size());
     EXPECT_EQ(output_data.size(), num_segments * rows_per_segment);
-    std::vector<uint32_t> segment_num_rows;
-    EXPECT_TRUE(output_rs_reader->get_segment_num_rows(&segment_num_rows).ok());
     // check vertical compaction result
     for (auto id = 0; id < output_data.size(); id++) {
         LOG(INFO) << "output data: " << std::get<0>(output_data[id]) << " "

From af8a975f9587ddd22687e866a9addf65a105ee52 Mon Sep 17 00:00:00 2001
From: minghong <zhouminghong@selectdb.com>
Date: Thu, 9 Jan 2025 15:19:41 +0800
Subject: [PATCH 075/140] [opt](nereids) infer in-predicate from or-predicate
 (#46468)

### What problem does this PR solve?
previous verion has follow drawbacks
1. inferred some in-predicates, which cannot be pushed down to storage
layer
2. it is easy to lead dead loop, because other expression rewrite rule
may remove its flag in expression's mutableState

in order to solve above issues, we implemented a new version
first, it is a plan node level rule to avoid to be applied to the same
expression repeatedly
second, we define replace mode and extract mode. if in replace mode, the
original expression should be equivalent to the inferred in-pred, which
is used for all plan node's expressions except filter.

for example, orig = "(a=1 and b=1) or (a=2 and c=2)" is equivalent to "a
in (1, 2) and (a=1 and b=1) or (a=2 and c=2)".
orig is a filter condition, "a in (1, 2)" can be pushed down to storage
layer, and this infer is useful. But if this is orig is an other join
condition, this inferrence is useless. So in extract mode, "a in (1, 2)"
is inferred, but in replace mode, it is not.
---
 .../doris/nereids/jobs/executor/Rewriter.java |   4 +
 .../apache/doris/nereids/rules/RuleType.java  |   1 +
 .../expression/ExpressionOptimization.java    |   2 -
 .../rules/expression/ExpressionRuleType.java  |   1 -
 .../rules/expression/rules/OrToIn.java        | 101 ++++++++++++------
 .../expression/rules/PartitionPruner.java     |   2 +-
 .../rules/rewrite/InferInPredicateFromOr.java |  91 ++++++++++++++++
 .../doris/nereids/util/MutableState.java      |   2 -
 .../expression/ExpressionRewriteTest.java     |  15 ---
 .../nereids/rules/rewrite/OrToInTest.java     |  99 +++++++++--------
 .../tpcds_sf100/noStatsRfPrune/query13.out    |   2 +-
 .../tpcds_sf100/noStatsRfPrune/query41.out    |   2 +-
 .../tpcds_sf100/no_stats_shape/query13.out    |   2 +-
 .../tpcds_sf100/no_stats_shape/query41.out    |   2 +-
 .../tpcds_sf100/rf_prune/query13.out          |   2 +-
 .../tpcds_sf100/rf_prune/query41.out          |   2 +-
 .../shape_check/tpcds_sf100/shape/query13.out |   2 +-
 .../shape_check/tpcds_sf100/shape/query41.out |   2 +-
 .../bs_downgrade_shape/query13.out            |   2 +-
 .../shape_check/tpcds_sf1000/hint/query13.out |   2 +-
 .../shape_check/tpcds_sf1000/hint/query41.out |   2 +-
 .../tpcds_sf1000/shape/query13.out            |   2 +-
 .../tpcds_sf1000/shape/query41.out            |   2 +-
 .../tpcds_sf10t_orc/shape/query13.out         |   2 +-
 .../tpcds_sf10t_orc/shape/query41.out         |   2 +-
 25 files changed, 229 insertions(+), 119 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferInPredicateFromOr.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
index 14b4d4e25388184..b1d1bd35a654a62 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
@@ -83,6 +83,7 @@
 import org.apache.doris.nereids.rules.rewrite.FindHashConditionForJoin;
 import org.apache.doris.nereids.rules.rewrite.InferAggNotNull;
 import org.apache.doris.nereids.rules.rewrite.InferFilterNotNull;
+import org.apache.doris.nereids.rules.rewrite.InferInPredicateFromOr;
 import org.apache.doris.nereids.rules.rewrite.InferJoinNotNull;
 import org.apache.doris.nereids.rules.rewrite.InferPredicates;
 import org.apache.doris.nereids.rules.rewrite.InferSetOperatorDistinct;
@@ -323,6 +324,9 @@ public class Rewriter extends AbstractBatchJobExecutor {
                         // after EliminateEmptyRelation, project can be pushed into union
                         topDown(new PushProjectIntoUnion())
                 ),
+                topic("infer In-predicate from Or-predicate",
+                        topDown(new InferInPredicateFromOr())
+                        ),
                 // putting the "Column pruning and infer predicate" topic behind the "Set operation optimization"
                 // is because that pulling up predicates from union needs EliminateEmptyRelation in union child
                 topic("Column pruning and infer predicate",
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index 7e012f7050f7bba..7d6b00912ee20ef 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -222,6 +222,7 @@ public enum RuleType {
     TRANSPOSE_LOGICAL_SEMI_JOIN_AGG_PROJECT(RuleTypeClass.REWRITE),
 
     // expression of plan rewrite
+    EXTRACT_IN_PREDICATE_FROM_OR(RuleTypeClass.REWRITE),
     REWRITE_ONE_ROW_RELATION_EXPRESSION(RuleTypeClass.REWRITE),
     REWRITE_PROJECT_EXPRESSION(RuleTypeClass.REWRITE),
     REWRITE_AGG_EXPRESSION(RuleTypeClass.REWRITE),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java
index de451daa049ac1f..1e76307327d22cf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java
@@ -26,7 +26,6 @@
 import org.apache.doris.nereids.rules.expression.rules.ExtractCommonFactorRule;
 import org.apache.doris.nereids.rules.expression.rules.LikeToEqualRewrite;
 import org.apache.doris.nereids.rules.expression.rules.NullSafeEqualToEqual;
-import org.apache.doris.nereids.rules.expression.rules.OrToIn;
 import org.apache.doris.nereids.rules.expression.rules.SimplifyComparisonPredicate;
 import org.apache.doris.nereids.rules.expression.rules.SimplifyInPredicate;
 import org.apache.doris.nereids.rules.expression.rules.SimplifyRange;
@@ -47,7 +46,6 @@ public class ExpressionOptimization extends ExpressionRewrite {
                     SimplifyComparisonPredicate.INSTANCE,
                     SimplifyInPredicate.INSTANCE,
                     SimplifyRange.INSTANCE,
-                    OrToIn.INSTANCE,
                     DateFunctionRewrite.INSTANCE,
                     ArrayContainToArrayOverlap.INSTANCE,
                     CaseWhenToIf.INSTANCE,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java
index d8919d1eaedc519..16881a61addb234 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java
@@ -40,7 +40,6 @@ public enum ExpressionRuleType {
     MERGE_DATE_TRUNC,
     NORMALIZE_BINARY_PREDICATES,
     NULL_SAFE_EQUAL_TO_EQUAL,
-    OR_TO_IN,
     REPLACE_VARIABLE_BY_LITERAL,
     SIMPLIFY_ARITHMETIC_COMPARISON,
     SIMPLIFY_ARITHMETIC,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OrToIn.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OrToIn.java
index 59b340478ce4324..136b40af5847c95 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OrToIn.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OrToIn.java
@@ -19,11 +19,8 @@
 
 import org.apache.doris.common.Pair;
 import org.apache.doris.nereids.rules.expression.ExpressionBottomUpRewriter;
-import org.apache.doris.nereids.rules.expression.ExpressionPatternMatcher;
-import org.apache.doris.nereids.rules.expression.ExpressionPatternRuleFactory;
 import org.apache.doris.nereids.rules.expression.ExpressionRewrite;
 import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
-import org.apache.doris.nereids.rules.expression.ExpressionRuleType;
 import org.apache.doris.nereids.trees.expressions.And;
 import org.apache.doris.nereids.trees.expressions.CompoundPredicate;
 import org.apache.doris.nereids.trees.expressions.EqualTo;
@@ -33,7 +30,6 @@
 import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.literal.Literal;
 import org.apache.doris.nereids.util.ExpressionUtils;
-import org.apache.doris.nereids.util.MutableState;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
@@ -45,48 +41,94 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 /**
- * dependends on SimplifyRange rule
+ * Do NOT use this rule in ExpressionOptimization
+ * apply this rule on filter expressions in extract mode,
+ * on other expressions in replace mode
  *
  */
-public class OrToIn implements ExpressionPatternRuleFactory {
+public class OrToIn {
+    /**
+     * case 1: from (a=1 and b=1) or (a=2), "a in (1, 2)" is inferred,
+     * inferred expr is not equivalent to the original expr
+     * - replaceMode: output origin expr
+     * - extractMode: output a in (1, 2) and  (a=1 and b=1) or (a=2)
+     *
+     * case 2: from (a=1) or (a=2), "a in (1,2)" is inferred, the inferred expr is equivalent to the original expr
+     * - replaceMode/extractMode: output a in (1, 2)
+     *
+     * extractMode only used for filter, the inferred In-predicate could be pushed down.
+     */
+    public enum Mode {
+        replaceMode,
+        extractMode
+    }
 
-    public static final OrToIn INSTANCE = new OrToIn();
+    public static final OrToIn EXTRACT_MODE_INSTANCE = new OrToIn(Mode.extractMode);
+    public static final OrToIn REPLACE_MODE_INSTANCE = new OrToIn(Mode.replaceMode);
 
     public static final int REWRITE_OR_TO_IN_PREDICATE_THRESHOLD = 2;
 
-    @Override
-    public List<ExpressionPatternMatcher<? extends Expression>> buildRules() {
-        return ImmutableList.of(
-                matchesTopType(Or.class).then(OrToIn.INSTANCE::rewrite)
-                        .toRule(ExpressionRuleType.OR_TO_IN)
-        );
+    private final Mode mode;
+
+    public OrToIn(Mode mode) {
+        this.mode = mode;
     }
 
+    /**
+     * simplify and then rewrite
+     */
     public Expression rewriteTree(Expression expr, ExpressionRewriteContext context) {
-        if (expr instanceof CompoundPredicate) {
-            expr = SimplifyRange.rewrite((CompoundPredicate) expr, context);
+        ExpressionBottomUpRewriter simplify = ExpressionRewrite.bottomUp(SimplifyRange.INSTANCE);
+        expr = simplify.rewrite(expr, context);
+        return rewriteTree(expr);
+
+    }
+
+    /**
+     * rewrite tree
+     */
+    public Expression rewriteTree(Expression expr) {
+        List<Expression> children = expr.children();
+        if (children.isEmpty()) {
+            return expr;
+        }
+        List<Expression> newChildren = children.stream()
+                .map(this::rewriteTree).collect(Collectors.toList());
+        if (expr instanceof And) {
+            // filter out duplicated conjunct
+            // example: OrToInTest.testDeDup()
+            Set<Expression> dedupSet = new LinkedHashSet<>();
+            for (Expression newChild : newChildren) {
+                dedupSet.addAll(ExpressionUtils.extractConjunction(newChild));
+            }
+            newChildren = Lists.newArrayList(dedupSet);
         }
-        ExpressionBottomUpRewriter bottomUpRewriter = ExpressionRewrite.bottomUp(this);
-        return bottomUpRewriter.rewrite(expr, context);
+        if (expr instanceof CompoundPredicate && newChildren.size() == 1) {
+            // (a=1) and (a=1)
+            // after rewrite, newChildren=[(a=1)]
+            expr = newChildren.get(0);
+        } else {
+            expr = expr.withChildren(newChildren);
+        }
+        if (expr instanceof Or) {
+            expr = rewrite((Or) expr);
+        }
+        return expr;
     }
 
     private Expression rewrite(Or or) {
-        if (or.getMutableState(MutableState.KEY_OR_TO_IN).isPresent()) {
-            return or;
-        }
         Pair<Expression, Expression> pair = extractCommonConjunct(or);
         Expression result = tryToRewriteIn(pair.second);
         if (pair.first != null) {
             result = new And(pair.first, result);
         }
-        result.setMutableState(MutableState.KEY_OR_TO_IN, 1);
         return result;
     }
 
     private Expression tryToRewriteIn(Expression or) {
-        or.setMutableState(MutableState.KEY_OR_TO_IN, 1);
         List<Expression> disjuncts = ExpressionUtils.extractDisjunction(or);
         for (Expression disjunct : disjuncts) {
             if (!hasInOrEqualChildren(disjunct)) {
@@ -114,7 +156,11 @@ private Expression tryToRewriteIn(Expression or) {
             Expression conjunct = candidatesToFinalResult(candidates);
             boolean keep = keepOriginalOrExpression(disjuncts);
             if (keep) {
-                return new And(conjunct, or);
+                if (mode == Mode.extractMode) {
+                    return new And(conjunct, or);
+                } else {
+                    return or;
+                }
             } else {
                 return conjunct;
             }
@@ -132,15 +178,6 @@ private boolean keepOriginalOrExpression(List<Expression> disjuncts) {
         return false;
     }
 
-    private boolean containsAny(Set a, Set b) {
-        for (Object x : a) {
-            if (b.contains(x)) {
-                return true;
-            }
-        }
-        return false;
-    }
-
     private Map<Expression, Set<Literal>> mergeCandidates(
             Map<Expression, Set<Literal>> a,
             Map<Expression, Set<Literal>> b) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
index 6fc3aabf25e21b9..fa81690afd3895c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
@@ -140,7 +140,7 @@ public static <K extends Comparable<K>> List<K> prune(List<Slot> partitionSlots,
                 "partitionPruningExpandThreshold",
                 10, sessionVariable -> sessionVariable.partitionPruningExpandThreshold);
 
-        partitionPredicate = OrToIn.INSTANCE.rewriteTree(
+        partitionPredicate = OrToIn.EXTRACT_MODE_INSTANCE.rewriteTree(
                 partitionPredicate, new ExpressionRewriteContext(cascadesContext));
         if (BooleanLiteral.TRUE.equals(partitionPredicate)) {
             return Utils.fastToImmutableList(idToPartitions.keySet());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferInPredicateFromOr.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferInPredicateFromOr.java
new file mode 100644
index 000000000000000..e798efbbde45a7e
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferInPredicateFromOr.java
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.rules.expression.rules.OrToIn;
+import org.apache.doris.nereids.trees.expressions.And;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.util.ExpressionUtils;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * infer In-predicate from Or
+ *
+ */
+public class InferInPredicateFromOr implements RewriteRuleFactory {
+
+    @Override
+    public List<Rule> buildRules() {
+        return ImmutableList.of(
+                logicalFilter().then(this::rewriteFilterExpression).toRule(RuleType.EXTRACT_IN_PREDICATE_FROM_OR),
+                logicalProject().then(this::rewriteProject).toRule(RuleType.EXTRACT_IN_PREDICATE_FROM_OR),
+                logicalJoin().whenNot(LogicalJoin::isMarkJoin)
+                        .then(this::rewriteJoin).toRule(RuleType.EXTRACT_IN_PREDICATE_FROM_OR)
+        );
+    }
+
+    private LogicalFilter<Plan> rewriteFilterExpression(LogicalFilter<Plan> filter) {
+        Expression rewritten = OrToIn.EXTRACT_MODE_INSTANCE.rewriteTree(filter.getPredicate());
+        Set<Expression> set = new LinkedHashSet<>(ExpressionUtils.extractConjunction(rewritten));
+        return filter.withConjuncts(set);
+    }
+
+    private LogicalProject<Plan> rewriteProject(LogicalProject<Plan> project) {
+        List<NamedExpression> newProjections = Lists.newArrayList();
+        for (NamedExpression proj : project.getProjects()) {
+            if (proj instanceof SlotReference) {
+                newProjections.add(proj);
+            } else {
+                Expression rewritten = OrToIn.REPLACE_MODE_INSTANCE.rewriteTree(proj);
+                newProjections.add((NamedExpression) rewritten);
+            }
+        }
+        return project.withProjects(newProjections);
+    }
+
+    private LogicalJoin<Plan, Plan> rewriteJoin(LogicalJoin<Plan, Plan> join) {
+        if (!join.isMarkJoin()) {
+            Expression otherCondition;
+            if (join.getOtherJoinConjuncts().isEmpty()) {
+                return join;
+            } else if (join.getOtherJoinConjuncts().size() == 1) {
+                otherCondition = join.getOtherJoinConjuncts().get(0);
+            } else {
+                otherCondition = new And(join.getOtherJoinConjuncts());
+            }
+            Expression rewritten = OrToIn.REPLACE_MODE_INSTANCE.rewriteTree(otherCondition);
+            join = join.withJoinConjuncts(join.getHashJoinConjuncts(), ExpressionUtils.extractConjunction(rewritten),
+                    join.getJoinReorderContext());
+        }
+        return join;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/MutableState.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/MutableState.java
index 528c19576bb40ee..16d7bb977610d77 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/MutableState.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/MutableState.java
@@ -29,8 +29,6 @@ public interface MutableState {
     String KEY_RF_JUMP = "rf-jump";
     String KEY_PUSH_TOPN_TO_AGG = "pushTopnToAgg";
 
-    String KEY_OR_TO_IN = "or_to_in";
-
     <T> Optional<T> get(String key);
 
     MutableState set(String key, Object value);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTest.java
index 13c345d099c6baf..13f2789c0a9ffc1 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTest.java
@@ -23,7 +23,6 @@
 import org.apache.doris.nereids.rules.expression.rules.InPredicateDedup;
 import org.apache.doris.nereids.rules.expression.rules.InPredicateToEqualToRule;
 import org.apache.doris.nereids.rules.expression.rules.NormalizeBinaryPredicatesRule;
-import org.apache.doris.nereids.rules.expression.rules.OrToIn;
 import org.apache.doris.nereids.rules.expression.rules.SimplifyCastRule;
 import org.apache.doris.nereids.rules.expression.rules.SimplifyComparisonPredicate;
 import org.apache.doris.nereids.rules.expression.rules.SimplifyNotExprRule;
@@ -169,20 +168,6 @@ void testExtractCommonFactorRewrite() {
 
     }
 
-    @Test
-    void testTpcdsCase() {
-        executor = new ExpressionRuleExecutor(ImmutableList.of(
-                bottomUp(
-                        SimplifyRange.INSTANCE,
-                        OrToIn.INSTANCE,
-                        ExtractCommonFactorRule.INSTANCE
-                )
-        ));
-        assertRewrite(
-                "(((((customer_address.ca_country = 'United States') AND ca_state IN ('DE', 'FL', 'TX')) OR ((customer_address.ca_country = 'United States') AND ca_state IN ('ID', 'IN', 'ND'))) OR ((customer_address.ca_country = 'United States') AND ca_state IN ('IL', 'MT', 'OH'))))",
-                "((customer_address.ca_country = 'United States') AND ca_state IN ('DE', 'FL', 'TX', 'ID', 'IN', 'ND', 'IL', 'MT', 'OH'))");
-    }
-
     @Test
     void testInPredicateToEqualToRule() {
         executor = new ExpressionRuleExecutor(ImmutableList.of(
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/OrToInTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/OrToInTest.java
index 774be68643feb2e..12f07dc77be6e86 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/OrToInTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/OrToInTest.java
@@ -20,58 +20,62 @@
 import org.apache.doris.nereids.rules.expression.ExpressionRewriteTestHelper;
 import org.apache.doris.nereids.rules.expression.rules.OrToIn;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.InPredicate;
-import org.apache.doris.nereids.trees.expressions.NamedExpression;
-import org.apache.doris.nereids.trees.expressions.literal.Literal;
 
-import com.google.common.collect.ImmutableSet;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
-import java.util.List;
-import java.util.Set;
-
 class OrToInTest extends ExpressionRewriteTestHelper {
 
+    @Test
+    void testDeDup() {
+        // (a=1 and b=1) or (a=2 and c=2) infers "a in (1, 2)"
+        // (a=1 and d=1) or (a=2 and e=2) infers "a in (1, 2)" again
+        // duplicated one should be removed
+        Expression expression = PARSER.parseExpression("((a=1 and b=1) or (a=2 and c=2)) and ((a=1 and d=1) or (a=2 and e=2))");
+        Expression rewritten = OrToIn.EXTRACT_MODE_INSTANCE.rewriteTree(expression, context);
+        Assertions.assertEquals("AND[a IN (1, 2),OR[AND[(a = 1),(b = 1)],AND[(a = 2),(c = 2)]],OR[AND[(a = 1),(d = 1)],AND[(a = 2),(e = 2)]]]",
+                rewritten.toSql());
+    }
+
+    // (a=1 and b=1) or (a=2 and c=2) infers "a in (1, 2)", but "a in (1, 2)" is not in root level,
+    // and cannot be push down, and hence, do not extract "a in (1, 2)"
+    @Test
+    void testExtractNothing() {
+        Expression expression = PARSER.parseExpression("((a=1 and b=1) or (a=2 and c=2) or d=1) and e = 2");
+        Expression rewritten = OrToIn.EXTRACT_MODE_INSTANCE.rewriteTree(expression, context);
+        Assertions.assertEquals("AND[OR[AND[(a = 1),(b = 1)],AND[(a = 2),(c = 2)],(d = 1)],(e = 2)]",
+                rewritten.toSql());
+    }
+
+    // replace mode
     @Test
     void test1() {
         String expr = "col1 = 1 or col1 = 2 or col1 = 3 and (col2 = 4)";
         Expression expression = PARSER.parseExpression(expr);
-        Expression rewritten = OrToIn.INSTANCE.rewriteTree(expression, context);
+        Expression rewritten = OrToIn.EXTRACT_MODE_INSTANCE.rewriteTree(expression, context);
         Assertions.assertEquals("AND[col1 IN (1, 2, 3),OR[col1 IN (1, 2),AND[(col1 = 3),(col2 = 4)]]]",
                 rewritten.toSql());
+        Expression rewritten2 = OrToIn.REPLACE_MODE_INSTANCE.rewriteTree(expression, context);
+        Assertions.assertEquals("OR[col1 IN (1, 2),AND[(col1 = 3),(col2 = 4)]]",
+                rewritten2.toSql());
     }
 
     @Test
     void test2() {
         String expr = "col1 = 1 and col1 = 3 and col2 = 3 or col2 = 4";
         Expression expression = PARSER.parseExpression(expr);
-        Expression rewritten = OrToIn.INSTANCE.rewriteTree(expression, context);
+        Expression rewritten = OrToIn.EXTRACT_MODE_INSTANCE.rewriteTree(expression, context);
         Assertions.assertEquals("(col2 = 4)",
                 rewritten.toSql());
     }
 
     @Test
     void test3() {
-        String expr = "(col1 = 1 or col1 = 2) and  (col2 = 3 or col2 = 4)";
-        Expression expression = PARSER.parseExpression(expr);
-        Expression rewritten = OrToIn.INSTANCE.rewriteTree(expression, context);
-        List<InPredicate> inPredicates = rewritten.collectToList(e -> e instanceof InPredicate);
-        Assertions.assertEquals(2, inPredicates.size());
-        InPredicate in1 = inPredicates.get(0);
-        Assertions.assertEquals("col1", ((NamedExpression) in1.getCompareExpr()).getName());
-        Set<Integer> opVals1 = ImmutableSet.of(1, 2);
-        for (Expression op : in1.getOptions()) {
-            Literal literal = (Literal) op;
-            Assertions.assertTrue(opVals1.contains(((Byte) literal.getValue()).intValue()));
-        }
-        InPredicate in2 = inPredicates.get(1);
-        Assertions.assertEquals("col2", ((NamedExpression) in2.getCompareExpr()).getName());
-        Set<Integer> opVals2 = ImmutableSet.of(3, 4);
-        for (Expression op : in2.getOptions()) {
-            Literal literal = (Literal) op;
-            Assertions.assertTrue(opVals2.contains(((Byte) literal.getValue()).intValue()));
-        }
+        String expr = "(A = 1 or A = 2) and  (B = 3 or B = 4)";
+        Expression expression = PARSER.parseExpression(expr);
+        Expression rewritten = OrToIn.EXTRACT_MODE_INSTANCE.rewriteTree(expression, context);
+        Assertions.assertEquals("AND[A IN (1, 2),B IN (3, 4)]",
+                rewritten.toSql());
     }
 
     @Test
@@ -79,7 +83,7 @@ void test4() {
         String expr = "case when col = 1 or col = 2 or col = 3 then 1"
                 + "         when col = 4 or col = 5 or col = 6 then 1 else 0 end";
         Expression expression = PARSER.parseExpression(expr);
-        Expression rewritten = OrToIn.INSTANCE.rewriteTree(expression, context);
+        Expression rewritten = OrToIn.REPLACE_MODE_INSTANCE.rewriteTree(expression, context);
         Assertions.assertEquals("CASE WHEN col IN (1, 2, 3) THEN 1 WHEN col IN (4, 5, 6) THEN 1 ELSE 0 END",
                 rewritten.toSql());
     }
@@ -88,7 +92,7 @@ void test4() {
     void test5() {
         String expr = "col = 1 or (col = 2 and (col = 3 or col = 4 or col = 5))";
         Expression expression = PARSER.parseExpression(expr);
-        Expression rewritten = OrToIn.INSTANCE.rewriteTree(expression, context);
+        Expression rewritten = OrToIn.EXTRACT_MODE_INSTANCE.rewriteTree(expression, context);
         Assertions.assertEquals("(col = 1)",
                 rewritten.toSql());
     }
@@ -97,7 +101,7 @@ void test5() {
     void test6() {
         String expr = "col = 1 or col = 2 or col in (1, 2, 3)";
         Expression expression = PARSER.parseExpression(expr);
-        Expression rewritten = OrToIn.INSTANCE.rewriteTree(expression, context);
+        Expression rewritten = OrToIn.EXTRACT_MODE_INSTANCE.rewriteTree(expression, context);
         Assertions.assertEquals("col IN (1, 2, 3)", rewritten.toSql());
     }
 
@@ -105,26 +109,17 @@ void test6() {
     void test7() {
         String expr = "A = 1 or A = 2 or abs(A)=5 or A in (1, 2, 3) or B = 1 or B = 2 or B in (1, 2, 3) or B+1 in (4, 5, 7)";
         Expression expression = PARSER.parseExpression(expr);
-        Expression rewritten = OrToIn.INSTANCE.rewriteTree(expression, context);
+        Expression rewritten = OrToIn.EXTRACT_MODE_INSTANCE.rewriteTree(expression, context);
         Assertions.assertEquals("OR[A IN (1, 2, 3),(abs(A) = 5),B IN (1, 2, 3),(B + 1) IN (4, 5, 7)]",
                 rewritten.toSql());
     }
 
-    @Test
-    void test8() {
-        String expr = "col = 1 or (col = 2 and (col = 3 or col = '4' or col = 5.0))";
-        Expression expression = PARSER.parseExpression(expr);
-        Expression rewritten = OrToIn.INSTANCE.rewriteTree(expression, context);
-        Assertions.assertEquals("OR[(col = 1),AND[(col = 2),col IN ('4', 3, 5.0)]]",
-                rewritten.toSql());
-    }
-
     @Test
     void testEnsureOrder() {
         // ensure not rewrite to col2 in (1, 2) or  cor 1 in (1, 2)
         String expr = "col1 IN (1, 2) OR col2 IN (1, 2)";
         Expression expression = PARSER.parseExpression(expr);
-        Expression rewritten = OrToIn.INSTANCE.rewriteTree(expression, context);
+        Expression rewritten = OrToIn.EXTRACT_MODE_INSTANCE.rewriteTree(expression, context);
         Assertions.assertEquals("OR[col1 IN (1, 2),col2 IN (1, 2)]",
                 rewritten.toSql());
     }
@@ -133,7 +128,7 @@ void testEnsureOrder() {
     void test9() {
         String expr = "col1=1 and (col2=1 or col2=2)";
         Expression expression = PARSER.parseExpression(expr);
-        Expression rewritten = OrToIn.INSTANCE.rewriteTree(expression, context);
+        Expression rewritten = OrToIn.EXTRACT_MODE_INSTANCE.rewriteTree(expression, context);
         Assertions.assertEquals("AND[(col1 = 1),col2 IN (1, 2)]",
                 rewritten.toSql());
     }
@@ -143,17 +138,18 @@ void test10() {
         // recursive rewrites
         String expr = "col1=1 or (col2 = 2 and (col3=4 or col3=5))";
         Expression expression = PARSER.parseExpression(expr);
-        Expression rewritten = OrToIn.INSTANCE.rewriteTree(expression, context);
+        Expression rewritten = OrToIn.EXTRACT_MODE_INSTANCE.rewriteTree(expression, context);
         Assertions.assertEquals("OR[(col1 = 1),AND[(col2 = 2),col3 IN (4, 5)]]",
                 rewritten.toSql());
     }
 
+    // replace mode
     @Test
     void test11() {
         // rewrite multi-inPredicates
         String expr = "(a=1 and b=2 and c=3) or (a=2 and b=2 and c=4)";
         Expression expression = PARSER.parseExpression(expr);
-        Expression rewritten = OrToIn.INSTANCE.rewriteTree(expression, context);
+        Expression rewritten = OrToIn.EXTRACT_MODE_INSTANCE.rewriteTree(expression, context);
         Assertions.assertEquals("AND[(b = 2),a IN (1, 2),c IN (3, 4),OR[AND[(a = 1),(c = 3)],AND[(a = 2),(c = 4)]]]",
                 rewritten.toSql());
     }
@@ -163,7 +159,7 @@ void test12() {
         // no rewrite
         String expr = "a in (1, 2) and a in (3, 4)";
         Expression expression = PARSER.parseExpression(expr);
-        Expression rewritten = OrToIn.INSTANCE.rewriteTree(expression, context);
+        Expression rewritten = OrToIn.EXTRACT_MODE_INSTANCE.rewriteTree(expression, context);
         Assertions.assertEquals("FALSE",
                 rewritten.toSql());
     }
@@ -173,7 +169,7 @@ void test13() {
         // no rewrite, because of "a like 'xyz'"
         String expr = "a like 'xyz% or a=1 or a=2': no extract";
         Expression expression = PARSER.parseExpression(expr);
-        Expression rewritten = OrToIn.INSTANCE.rewriteTree(expression, context);
+        Expression rewritten = OrToIn.EXTRACT_MODE_INSTANCE.rewriteTree(expression, context);
         Assertions.assertEquals("(a like 'xyz% or a=1 or a=2')",
                 rewritten.toSql());
     }
@@ -183,7 +179,7 @@ void test14() {
         // no rewrite, because of "f(a)"
         String expr = "(a=1 and f(a)=2) or a=3";
         Expression expression = PARSER.parseExpression(expr);
-        Expression rewritten = OrToIn.INSTANCE.rewriteTree(expression, context);
+        Expression rewritten = OrToIn.EXTRACT_MODE_INSTANCE.rewriteTree(expression, context);
         Assertions.assertEquals("OR[AND[(a = 1),(f(a) = 2)],(a = 3)]",
                 rewritten.toSql());
     }
@@ -193,7 +189,7 @@ void test15() {
         // no rewrite, because of "a like 'xyz'"
         String expr = "x=1 or (a=1 and b=2) or (a=2 and c=3)";
         Expression expression = PARSER.parseExpression(expr);
-        Expression rewritten = OrToIn.INSTANCE.rewriteTree(expression, context);
+        Expression rewritten = OrToIn.EXTRACT_MODE_INSTANCE.rewriteTree(expression, context);
         Assertions.assertEquals("OR[(x = 1),AND[(a = 1),(b = 2)],AND[(a = 2),(c = 3)]]",
                 rewritten.toSql());
     }
@@ -202,16 +198,17 @@ void test15() {
     void test16() {
         String expr = "a=1 or a=1 or a=1";
         Expression expression = PARSER.parseExpression(expr);
-        Expression rewritten = OrToIn.INSTANCE.rewriteTree(expression, context);
+        Expression rewritten = OrToIn.EXTRACT_MODE_INSTANCE.rewriteTree(expression, context);
         Assertions.assertEquals("(a = 1)",
                 rewritten.toSql());
     }
 
+    //replace mode
     @Test
     void test17() {
         String expr = "(a=1 and b=2) or (a in (2, 3) and ((a=2 and c=3) or (a=3 and d=4)))";
         Expression expression = PARSER.parseExpression(expr);
-        Expression rewritten = OrToIn.INSTANCE.rewriteTree(expression, context);
+        Expression rewritten = OrToIn.EXTRACT_MODE_INSTANCE.rewriteTree(expression, context);
         System.out.println(rewritten);
     }
 }
diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query13.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query13.out
index 7b5d0d01ba6a175..7530c5b63e152f4 100644
--- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query13.out
+++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query13.out
@@ -9,7 +9,7 @@ PhysicalResultSink
 ------------PhysicalProject
 --------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->[ss_addr_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('M', 'S'),cd_education_status IN ('4 yr Degree', 'College'),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=() build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query41.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query41.out
index 3034a77fe0897a4..8ed17f7cd88452f 100644
--- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query41.out
+++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter(OR[AND[i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet'),i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp'),OR[AND[(item.i_category = 'Women'),i_color IN ('aquamarine', 'gainsboro'),i_units IN ('Dozen', 'Ounce'),i_size IN ('economy', 'medium')],AND[(item.i_category = 'Women'),i_color IN ('chiffon', 'violet'),i_units IN ('Pound', 'Ton'),i_size IN ('extra large', 'small')],AND[(item.i_category = 'Men'),i_color IN ('blue', 'chartreuse'),i_units IN ('Each', 'Oz'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('dodger', 'tan'),i_units IN ('Bunch', 'Tsp'),i_size IN ('economy', 'medium')]]],AND[i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato'),i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown'),OR[AND[(item.i_category = 'Women'),i_color IN ('blanched', 'tomato'),i_units IN ('Case', 'Tbl'),i_size IN ('economy', 'medium')],AND[(item.i_category = 'Women'),i_color IN ('almond', 'lime'),i_units IN ('Box', 'Dram'),i_size IN ('extra large', 'small')],AND[(item.i_category = 'Men'),i_color IN ('peru', 'saddle'),i_units IN ('Gram', 'Pallet'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('indian', 'spring'),i_units IN ('Carton', 'Unknown'),i_size IN ('economy', 'medium')]]]] and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
+------------------------------filter(OR[AND[(item.i_category = 'Men'),i_size IN ('N/A', 'economy', 'large', 'medium'),i_color IN ('blue', 'chartreuse', 'dodger', 'indian', 'peru', 'saddle', 'spring', 'tan'),i_units IN ('Bunch', 'Carton', 'Each', 'Gram', 'Oz', 'Pallet', 'Tsp', 'Unknown'),OR[AND[i_size IN ('economy', 'medium'),i_color IN ('dodger', 'tan'),i_units IN ('Bunch', 'Tsp')],AND[i_color IN ('blue', 'chartreuse'),i_units IN ('Each', 'Oz'),i_size IN ('N/A', 'large')],AND[i_size IN ('economy', 'medium'),i_color IN ('indian', 'spring'),i_units IN ('Carton', 'Unknown')],AND[i_color IN ('peru', 'saddle'),i_units IN ('Gram', 'Pallet'),i_size IN ('N/A', 'large')]]],AND[(item.i_category = 'Women'),i_color IN ('almond', 'aquamarine', 'blanched', 'chiffon', 'gainsboro', 'lime', 'tomato', 'violet'),i_units IN ('Box', 'Case', 'Dozen', 'Dram', 'Ounce', 'Pound', 'Tbl', 'Ton'),i_size IN ('economy', 'extra large', 'medium', 'small'),OR[AND[i_color IN ('aquamarine', 'gainsboro'),i_units IN ('Dozen', 'Ounce'),i_size IN ('economy', 'medium')],AND[i_color IN ('chiffon', 'violet'),i_units IN ('Pound', 'Ton'),i_size IN ('extra large', 'small')],AND[i_color IN ('blanched', 'tomato'),i_units IN ('Case', 'Tbl'),i_size IN ('economy', 'medium')],AND[i_color IN ('almond', 'lime'),i_units IN ('Box', 'Dram'),i_size IN ('extra large', 'small')]]]] and i_category IN ('Men', 'Women'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query13.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query13.out
index 2cac8d809ad1245..a77c007b00f8b43 100644
--- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query13.out
+++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query13.out
@@ -9,7 +9,7 @@ PhysicalResultSink
 ------------PhysicalProject
 --------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->[ss_addr_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('M', 'S'),cd_education_status IN ('4 yr Degree', 'College'),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=() build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query41.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query41.out
index 3034a77fe0897a4..8ed17f7cd88452f 100644
--- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query41.out
+++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter(OR[AND[i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet'),i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp'),OR[AND[(item.i_category = 'Women'),i_color IN ('aquamarine', 'gainsboro'),i_units IN ('Dozen', 'Ounce'),i_size IN ('economy', 'medium')],AND[(item.i_category = 'Women'),i_color IN ('chiffon', 'violet'),i_units IN ('Pound', 'Ton'),i_size IN ('extra large', 'small')],AND[(item.i_category = 'Men'),i_color IN ('blue', 'chartreuse'),i_units IN ('Each', 'Oz'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('dodger', 'tan'),i_units IN ('Bunch', 'Tsp'),i_size IN ('economy', 'medium')]]],AND[i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato'),i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown'),OR[AND[(item.i_category = 'Women'),i_color IN ('blanched', 'tomato'),i_units IN ('Case', 'Tbl'),i_size IN ('economy', 'medium')],AND[(item.i_category = 'Women'),i_color IN ('almond', 'lime'),i_units IN ('Box', 'Dram'),i_size IN ('extra large', 'small')],AND[(item.i_category = 'Men'),i_color IN ('peru', 'saddle'),i_units IN ('Gram', 'Pallet'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('indian', 'spring'),i_units IN ('Carton', 'Unknown'),i_size IN ('economy', 'medium')]]]] and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
+------------------------------filter(OR[AND[(item.i_category = 'Men'),i_size IN ('N/A', 'economy', 'large', 'medium'),i_color IN ('blue', 'chartreuse', 'dodger', 'indian', 'peru', 'saddle', 'spring', 'tan'),i_units IN ('Bunch', 'Carton', 'Each', 'Gram', 'Oz', 'Pallet', 'Tsp', 'Unknown'),OR[AND[i_size IN ('economy', 'medium'),i_color IN ('dodger', 'tan'),i_units IN ('Bunch', 'Tsp')],AND[i_color IN ('blue', 'chartreuse'),i_units IN ('Each', 'Oz'),i_size IN ('N/A', 'large')],AND[i_size IN ('economy', 'medium'),i_color IN ('indian', 'spring'),i_units IN ('Carton', 'Unknown')],AND[i_color IN ('peru', 'saddle'),i_units IN ('Gram', 'Pallet'),i_size IN ('N/A', 'large')]]],AND[(item.i_category = 'Women'),i_color IN ('almond', 'aquamarine', 'blanched', 'chiffon', 'gainsboro', 'lime', 'tomato', 'violet'),i_units IN ('Box', 'Case', 'Dozen', 'Dram', 'Ounce', 'Pound', 'Tbl', 'Ton'),i_size IN ('economy', 'extra large', 'medium', 'small'),OR[AND[i_color IN ('aquamarine', 'gainsboro'),i_units IN ('Dozen', 'Ounce'),i_size IN ('economy', 'medium')],AND[i_color IN ('chiffon', 'violet'),i_units IN ('Pound', 'Ton'),i_size IN ('extra large', 'small')],AND[i_color IN ('blanched', 'tomato'),i_units IN ('Case', 'Tbl'),i_size IN ('economy', 'medium')],AND[i_color IN ('almond', 'lime'),i_units IN ('Box', 'Dram'),i_size IN ('extra large', 'small')]]]] and i_category IN ('Men', 'Women'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query13.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query13.out
index 55cadf71bf5d457..2c3c76c20ccd96f 100644
--- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query13.out
+++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query13.out
@@ -7,7 +7,7 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=()
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('M', 'S'),cd_education_status IN ('4 yr Degree', 'College'),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF3 ss_cdemo_sk->[cd_demo_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF3 ss_cdemo_sk->[cd_demo_sk]
 ----------------PhysicalProject
 ------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree')]] and cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and cd_marital_status IN ('D', 'M', 'S'))
 --------------------PhysicalOlapScan[customer_demographics] apply RFs: RF3
diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query41.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query41.out
index 3034a77fe0897a4..8ed17f7cd88452f 100644
--- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query41.out
+++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter(OR[AND[i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet'),i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp'),OR[AND[(item.i_category = 'Women'),i_color IN ('aquamarine', 'gainsboro'),i_units IN ('Dozen', 'Ounce'),i_size IN ('economy', 'medium')],AND[(item.i_category = 'Women'),i_color IN ('chiffon', 'violet'),i_units IN ('Pound', 'Ton'),i_size IN ('extra large', 'small')],AND[(item.i_category = 'Men'),i_color IN ('blue', 'chartreuse'),i_units IN ('Each', 'Oz'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('dodger', 'tan'),i_units IN ('Bunch', 'Tsp'),i_size IN ('economy', 'medium')]]],AND[i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato'),i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown'),OR[AND[(item.i_category = 'Women'),i_color IN ('blanched', 'tomato'),i_units IN ('Case', 'Tbl'),i_size IN ('economy', 'medium')],AND[(item.i_category = 'Women'),i_color IN ('almond', 'lime'),i_units IN ('Box', 'Dram'),i_size IN ('extra large', 'small')],AND[(item.i_category = 'Men'),i_color IN ('peru', 'saddle'),i_units IN ('Gram', 'Pallet'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('indian', 'spring'),i_units IN ('Carton', 'Unknown'),i_size IN ('economy', 'medium')]]]] and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
+------------------------------filter(OR[AND[(item.i_category = 'Men'),i_size IN ('N/A', 'economy', 'large', 'medium'),i_color IN ('blue', 'chartreuse', 'dodger', 'indian', 'peru', 'saddle', 'spring', 'tan'),i_units IN ('Bunch', 'Carton', 'Each', 'Gram', 'Oz', 'Pallet', 'Tsp', 'Unknown'),OR[AND[i_size IN ('economy', 'medium'),i_color IN ('dodger', 'tan'),i_units IN ('Bunch', 'Tsp')],AND[i_color IN ('blue', 'chartreuse'),i_units IN ('Each', 'Oz'),i_size IN ('N/A', 'large')],AND[i_size IN ('economy', 'medium'),i_color IN ('indian', 'spring'),i_units IN ('Carton', 'Unknown')],AND[i_color IN ('peru', 'saddle'),i_units IN ('Gram', 'Pallet'),i_size IN ('N/A', 'large')]]],AND[(item.i_category = 'Women'),i_color IN ('almond', 'aquamarine', 'blanched', 'chiffon', 'gainsboro', 'lime', 'tomato', 'violet'),i_units IN ('Box', 'Case', 'Dozen', 'Dram', 'Ounce', 'Pound', 'Tbl', 'Ton'),i_size IN ('economy', 'extra large', 'medium', 'small'),OR[AND[i_color IN ('aquamarine', 'gainsboro'),i_units IN ('Dozen', 'Ounce'),i_size IN ('economy', 'medium')],AND[i_color IN ('chiffon', 'violet'),i_units IN ('Pound', 'Ton'),i_size IN ('extra large', 'small')],AND[i_color IN ('blanched', 'tomato'),i_units IN ('Case', 'Tbl'),i_size IN ('economy', 'medium')],AND[i_color IN ('almond', 'lime'),i_units IN ('Box', 'Dram'),i_size IN ('extra large', 'small')]]]] and i_category IN ('Men', 'Women'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query13.out b/regression-test/data/shape_check/tpcds_sf100/shape/query13.out
index dc2239ca3f9701b..4c12073a11d3209 100644
--- a/regression-test/data/shape_check/tpcds_sf100/shape/query13.out
+++ b/regression-test/data/shape_check/tpcds_sf100/shape/query13.out
@@ -7,7 +7,7 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF4 s_store_sk->[ss_store_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('M', 'S'),cd_education_status IN ('4 yr Degree', 'College'),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF3 ss_cdemo_sk->[cd_demo_sk]
+--------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF3 ss_cdemo_sk->[cd_demo_sk]
 ----------------PhysicalProject
 ------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree')]] and cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and cd_marital_status IN ('D', 'M', 'S'))
 --------------------PhysicalOlapScan[customer_demographics] apply RFs: RF3
diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query41.out b/regression-test/data/shape_check/tpcds_sf100/shape/query41.out
index 3034a77fe0897a4..8ed17f7cd88452f 100644
--- a/regression-test/data/shape_check/tpcds_sf100/shape/query41.out
+++ b/regression-test/data/shape_check/tpcds_sf100/shape/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter(OR[AND[i_color IN ('aquamarine', 'blue', 'chartreuse', 'chiffon', 'dodger', 'gainsboro', 'tan', 'violet'),i_units IN ('Bunch', 'Dozen', 'Each', 'Ounce', 'Oz', 'Pound', 'Ton', 'Tsp'),OR[AND[(item.i_category = 'Women'),i_color IN ('aquamarine', 'gainsboro'),i_units IN ('Dozen', 'Ounce'),i_size IN ('economy', 'medium')],AND[(item.i_category = 'Women'),i_color IN ('chiffon', 'violet'),i_units IN ('Pound', 'Ton'),i_size IN ('extra large', 'small')],AND[(item.i_category = 'Men'),i_color IN ('blue', 'chartreuse'),i_units IN ('Each', 'Oz'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('dodger', 'tan'),i_units IN ('Bunch', 'Tsp'),i_size IN ('economy', 'medium')]]],AND[i_color IN ('almond', 'blanched', 'indian', 'lime', 'peru', 'saddle', 'spring', 'tomato'),i_units IN ('Box', 'Carton', 'Case', 'Dram', 'Gram', 'Pallet', 'Tbl', 'Unknown'),OR[AND[(item.i_category = 'Women'),i_color IN ('blanched', 'tomato'),i_units IN ('Case', 'Tbl'),i_size IN ('economy', 'medium')],AND[(item.i_category = 'Women'),i_color IN ('almond', 'lime'),i_units IN ('Box', 'Dram'),i_size IN ('extra large', 'small')],AND[(item.i_category = 'Men'),i_color IN ('peru', 'saddle'),i_units IN ('Gram', 'Pallet'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('indian', 'spring'),i_units IN ('Carton', 'Unknown'),i_size IN ('economy', 'medium')]]]] and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'medium', 'small'))
+------------------------------filter(OR[AND[(item.i_category = 'Men'),i_size IN ('N/A', 'economy', 'large', 'medium'),i_color IN ('blue', 'chartreuse', 'dodger', 'indian', 'peru', 'saddle', 'spring', 'tan'),i_units IN ('Bunch', 'Carton', 'Each', 'Gram', 'Oz', 'Pallet', 'Tsp', 'Unknown'),OR[AND[i_size IN ('economy', 'medium'),i_color IN ('dodger', 'tan'),i_units IN ('Bunch', 'Tsp')],AND[i_color IN ('blue', 'chartreuse'),i_units IN ('Each', 'Oz'),i_size IN ('N/A', 'large')],AND[i_size IN ('economy', 'medium'),i_color IN ('indian', 'spring'),i_units IN ('Carton', 'Unknown')],AND[i_color IN ('peru', 'saddle'),i_units IN ('Gram', 'Pallet'),i_size IN ('N/A', 'large')]]],AND[(item.i_category = 'Women'),i_color IN ('almond', 'aquamarine', 'blanched', 'chiffon', 'gainsboro', 'lime', 'tomato', 'violet'),i_units IN ('Box', 'Case', 'Dozen', 'Dram', 'Ounce', 'Pound', 'Tbl', 'Ton'),i_size IN ('economy', 'extra large', 'medium', 'small'),OR[AND[i_color IN ('aquamarine', 'gainsboro'),i_units IN ('Dozen', 'Ounce'),i_size IN ('economy', 'medium')],AND[i_color IN ('chiffon', 'violet'),i_units IN ('Pound', 'Ton'),i_size IN ('extra large', 'small')],AND[i_color IN ('blanched', 'tomato'),i_units IN ('Case', 'Tbl'),i_size IN ('economy', 'medium')],AND[i_color IN ('almond', 'lime'),i_units IN ('Box', 'Dram'),i_size IN ('extra large', 'small')]]]] and i_category IN ('Men', 'Women'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query13.out b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query13.out
index 0db3330914c18ad..036782979d7fe89 100644
--- a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query13.out
+++ b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query13.out
@@ -14,7 +14,7 @@ PhysicalResultSink
 ----------------PhysicalProject
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
 --------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('D', 'W'),cd_education_status IN ('2 yr Degree', 'Primary'),OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
 ------------------------PhysicalProject
 --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=() build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ----------------------------PhysicalProject
diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query13.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query13.out
index 91efc8daefd8581..7baf0f98b8e82f8 100644
--- a/regression-test/data/shape_check/tpcds_sf1000/hint/query13.out
+++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query13.out
@@ -7,7 +7,7 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF4 s_store_sk->[ss_store_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('D', 'W'),cd_education_status IN ('2 yr Degree', 'Primary'),OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF3 ss_cdemo_sk->[cd_demo_sk]
+--------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF3 ss_cdemo_sk->[cd_demo_sk]
 ----------------PhysicalProject
 ------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree')]] and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'M', 'W'))
 --------------------PhysicalOlapScan[customer_demographics] apply RFs: RF3
diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query41.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query41.out
index 0bba60d4cdac394..a5ebf7e0dd61d7f 100644
--- a/regression-test/data/shape_check/tpcds_sf1000/hint/query41.out
+++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter(OR[AND[i_color IN ('forest', 'lime', 'maroon', 'navy', 'powder', 'sky', 'slate', 'smoke'),i_units IN ('Bunch', 'Case', 'Dozen', 'Gross', 'Lb', 'Ounce', 'Pallet', 'Pound'),OR[AND[(item.i_category = 'Women'),i_color IN ('forest', 'lime'),i_units IN ('Pallet', 'Pound'),i_size IN ('economy', 'small')],AND[(item.i_category = 'Women'),i_color IN ('navy', 'slate'),i_units IN ('Bunch', 'Gross'),i_size IN ('extra large', 'petite')],AND[(item.i_category = 'Men'),i_color IN ('powder', 'sky'),i_units IN ('Dozen', 'Lb'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('maroon', 'smoke'),i_units IN ('Case', 'Ounce'),i_size IN ('economy', 'small')]]],AND[i_color IN ('aquamarine', 'dark', 'firebrick', 'frosted', 'papaya', 'peach', 'plum', 'sienna'),i_units IN ('Box', 'Bundle', 'Carton', 'Cup', 'Dram', 'Each', 'Tbl', 'Ton'),OR[AND[(item.i_category = 'Women'),i_color IN ('aquamarine', 'dark'),i_units IN ('Tbl', 'Ton'),i_size IN ('economy', 'small')],AND[(item.i_category = 'Women'),i_color IN ('frosted', 'plum'),i_units IN ('Box', 'Dram'),i_size IN ('extra large', 'petite')],AND[(item.i_category = 'Men'),i_color IN ('papaya', 'peach'),i_units IN ('Bundle', 'Carton'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('firebrick', 'sienna'),i_units IN ('Cup', 'Each'),i_size IN ('economy', 'small')]]]] and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'petite', 'small'))
+------------------------------filter(OR[AND[(item.i_category = 'Men'),i_size IN ('N/A', 'economy', 'large', 'small'),i_color IN ('firebrick', 'maroon', 'papaya', 'peach', 'powder', 'sienna', 'sky', 'smoke'),i_units IN ('Bundle', 'Carton', 'Case', 'Cup', 'Dozen', 'Each', 'Lb', 'Ounce'),OR[AND[i_size IN ('economy', 'small'),i_color IN ('maroon', 'smoke'),i_units IN ('Case', 'Ounce')],AND[i_color IN ('powder', 'sky'),i_units IN ('Dozen', 'Lb'),i_size IN ('N/A', 'large')],AND[i_size IN ('economy', 'small'),i_color IN ('firebrick', 'sienna'),i_units IN ('Cup', 'Each')],AND[i_color IN ('papaya', 'peach'),i_units IN ('Bundle', 'Carton'),i_size IN ('N/A', 'large')]]],AND[(item.i_category = 'Women'),i_color IN ('aquamarine', 'dark', 'forest', 'frosted', 'lime', 'navy', 'plum', 'slate'),i_units IN ('Box', 'Bunch', 'Dram', 'Gross', 'Pallet', 'Pound', 'Tbl', 'Ton'),i_size IN ('economy', 'extra large', 'petite', 'small'),OR[AND[i_color IN ('forest', 'lime'),i_units IN ('Pallet', 'Pound'),i_size IN ('economy', 'small')],AND[i_color IN ('navy', 'slate'),i_units IN ('Bunch', 'Gross'),i_size IN ('extra large', 'petite')],AND[i_color IN ('aquamarine', 'dark'),i_units IN ('Tbl', 'Ton'),i_size IN ('economy', 'small')],AND[i_color IN ('frosted', 'plum'),i_units IN ('Box', 'Dram'),i_size IN ('extra large', 'petite')]]]] and i_category IN ('Men', 'Women'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query13.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query13.out
index 0db3330914c18ad..036782979d7fe89 100644
--- a/regression-test/data/shape_check/tpcds_sf1000/shape/query13.out
+++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query13.out
@@ -14,7 +14,7 @@ PhysicalResultSink
 ----------------PhysicalProject
 ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
 --------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('D', 'W'),cd_education_status IN ('2 yr Degree', 'Primary'),OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
+----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
 ------------------------PhysicalProject
 --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=() build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ----------------------------PhysicalProject
diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query41.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query41.out
index 0bba60d4cdac394..a5ebf7e0dd61d7f 100644
--- a/regression-test/data/shape_check/tpcds_sf1000/shape/query41.out
+++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter(OR[AND[i_color IN ('forest', 'lime', 'maroon', 'navy', 'powder', 'sky', 'slate', 'smoke'),i_units IN ('Bunch', 'Case', 'Dozen', 'Gross', 'Lb', 'Ounce', 'Pallet', 'Pound'),OR[AND[(item.i_category = 'Women'),i_color IN ('forest', 'lime'),i_units IN ('Pallet', 'Pound'),i_size IN ('economy', 'small')],AND[(item.i_category = 'Women'),i_color IN ('navy', 'slate'),i_units IN ('Bunch', 'Gross'),i_size IN ('extra large', 'petite')],AND[(item.i_category = 'Men'),i_color IN ('powder', 'sky'),i_units IN ('Dozen', 'Lb'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('maroon', 'smoke'),i_units IN ('Case', 'Ounce'),i_size IN ('economy', 'small')]]],AND[i_color IN ('aquamarine', 'dark', 'firebrick', 'frosted', 'papaya', 'peach', 'plum', 'sienna'),i_units IN ('Box', 'Bundle', 'Carton', 'Cup', 'Dram', 'Each', 'Tbl', 'Ton'),OR[AND[(item.i_category = 'Women'),i_color IN ('aquamarine', 'dark'),i_units IN ('Tbl', 'Ton'),i_size IN ('economy', 'small')],AND[(item.i_category = 'Women'),i_color IN ('frosted', 'plum'),i_units IN ('Box', 'Dram'),i_size IN ('extra large', 'petite')],AND[(item.i_category = 'Men'),i_color IN ('papaya', 'peach'),i_units IN ('Bundle', 'Carton'),i_size IN ('N/A', 'large')],AND[(item.i_category = 'Men'),i_color IN ('firebrick', 'sienna'),i_units IN ('Cup', 'Each'),i_size IN ('economy', 'small')]]]] and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'extra large', 'large', 'petite', 'small'))
+------------------------------filter(OR[AND[(item.i_category = 'Men'),i_size IN ('N/A', 'economy', 'large', 'small'),i_color IN ('firebrick', 'maroon', 'papaya', 'peach', 'powder', 'sienna', 'sky', 'smoke'),i_units IN ('Bundle', 'Carton', 'Case', 'Cup', 'Dozen', 'Each', 'Lb', 'Ounce'),OR[AND[i_size IN ('economy', 'small'),i_color IN ('maroon', 'smoke'),i_units IN ('Case', 'Ounce')],AND[i_color IN ('powder', 'sky'),i_units IN ('Dozen', 'Lb'),i_size IN ('N/A', 'large')],AND[i_size IN ('economy', 'small'),i_color IN ('firebrick', 'sienna'),i_units IN ('Cup', 'Each')],AND[i_color IN ('papaya', 'peach'),i_units IN ('Bundle', 'Carton'),i_size IN ('N/A', 'large')]]],AND[(item.i_category = 'Women'),i_color IN ('aquamarine', 'dark', 'forest', 'frosted', 'lime', 'navy', 'plum', 'slate'),i_units IN ('Box', 'Bunch', 'Dram', 'Gross', 'Pallet', 'Pound', 'Tbl', 'Ton'),i_size IN ('economy', 'extra large', 'petite', 'small'),OR[AND[i_color IN ('forest', 'lime'),i_units IN ('Pallet', 'Pound'),i_size IN ('economy', 'small')],AND[i_color IN ('navy', 'slate'),i_units IN ('Bunch', 'Gross'),i_size IN ('extra large', 'petite')],AND[i_color IN ('aquamarine', 'dark'),i_units IN ('Tbl', 'Ton'),i_size IN ('economy', 'small')],AND[i_color IN ('frosted', 'plum'),i_units IN ('Box', 'Dram'),i_size IN ('extra large', 'petite')]]]] and i_category IN ('Men', 'Women'))
 --------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query13.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query13.out
index 8cf5a0dbc062aa4..13791d12b9c0c7a 100644
--- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query13.out
+++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query13.out
@@ -9,7 +9,7 @@ PhysicalResultSink
 ------------PhysicalProject
 --------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('MI', 'OK', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('NC', 'OH', 'WA'),(store_sales.ss_net_profit >= 150.00)],AND[ca_state IN ('FL', 'GA', 'MT'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->[ss_addr_sk]
 ----------------PhysicalProject
-------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),cd_marital_status IN ('D', 'W'),cd_education_status IN ('College', 'Primary'),OR[AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Secondary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
+------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),OR[AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Secondary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF2 hd_demo_sk->[ss_hdemo_sk]
 --------------------PhysicalProject
 ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=() build RFs:RF1 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query41.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query41.out
index c99d2b2031c4eac..25490422b6ea7d0 100644
--- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query41.out
+++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query41.out
@@ -18,6 +18,6 @@ PhysicalResultSink
 ------------------------PhysicalDistribute[DistributionSpecHash]
 --------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------filter(OR[AND[i_color IN ('black', 'chocolate', 'cornflower', 'firebrick', 'frosted', 'magenta', 'rose', 'slate'),i_units IN ('Box', 'Bundle', 'Carton', 'Dram', 'Gross', 'Lb', 'Oz', 'Pound'),OR[AND[(item.i_category = 'Women'),i_color IN ('frosted', 'rose'),i_units IN ('Gross', 'Lb'),i_size IN ('large', 'medium')],AND[(item.i_category = 'Women'),i_color IN ('black', 'chocolate'),i_units IN ('Box', 'Dram'),i_size IN ('economy', 'petite')],AND[(item.i_category = 'Men'),i_color IN ('magenta', 'slate'),i_units IN ('Bundle', 'Carton'),i_size IN ('N/A', 'small')],AND[(item.i_category = 'Men'),i_color IN ('cornflower', 'firebrick'),i_units IN ('Oz', 'Pound'),i_size IN ('large', 'medium')]]],AND[i_color IN ('almond', 'aquamarine', 'cyan', 'lavender', 'maroon', 'papaya', 'purple', 'steel'),i_units IN ('Bunch', 'Case', 'Cup', 'Each', 'Gram', 'N/A', 'Pallet', 'Tsp'),OR[AND[(item.i_category = 'Women'),i_color IN ('almond', 'steel'),i_units IN ('Case', 'Tsp'),i_size IN ('large', 'medium')],AND[(item.i_category = 'Women'),i_color IN ('aquamarine', 'purple'),i_units IN ('Bunch', 'Gram'),i_size IN ('economy', 'petite')],AND[(item.i_category = 'Men'),i_color IN ('lavender', 'papaya'),i_units IN ('Cup', 'Pallet'),i_size IN ('N/A', 'small')],AND[(item.i_category = 'Men'),i_color IN ('cyan', 'maroon'),i_units IN ('Each', 'N/A'),i_size IN ('large', 'medium')]]]] and i_category IN ('Men', 'Women') and i_size IN ('N/A', 'economy', 'large', 'medium', 'petite', 'small'))
+------------------------------filter(OR[AND[(item.i_category = 'Men'),i_size IN ('N/A', 'large', 'medium', 'small'),i_color IN ('cornflower', 'cyan', 'firebrick', 'lavender', 'magenta', 'maroon', 'papaya', 'slate'),i_units IN ('Bundle', 'Carton', 'Cup', 'Each', 'N/A', 'Oz', 'Pallet', 'Pound'),OR[AND[i_size IN ('large', 'medium'),i_color IN ('cornflower', 'firebrick'),i_units IN ('Oz', 'Pound')],AND[i_color IN ('magenta', 'slate'),i_units IN ('Bundle', 'Carton'),i_size IN ('N/A', 'small')],AND[i_size IN ('large', 'medium'),i_color IN ('cyan', 'maroon'),i_units IN ('Each', 'N/A')],AND[i_color IN ('lavender', 'papaya'),i_units IN ('Cup', 'Pallet'),i_size IN ('N/A', 'small')]]],AND[(item.i_category = 'Women'),i_color IN ('almond', 'aquamarine', 'black', 'chocolate', 'frosted', 'purple', 'rose', 'steel'),i_units IN ('Box', 'Bunch', 'Case', 'Dram', 'Gram', 'Gross', 'Lb', 'Tsp'),i_size IN ('economy', 'large', 'medium', 'petite'),OR[AND[i_color IN ('frosted', 'rose'),i_units IN ('Gross', 'Lb'),i_size IN ('large', 'medium')],AND[i_color IN ('black', 'chocolate'),i_units IN ('Box', 'Dram'),i_size IN ('economy', 'petite')],AND[i_color IN ('almond', 'steel'),i_units IN ('Case', 'Tsp'),i_size IN ('large', 'medium')],AND[i_color IN ('aquamarine', 'purple'),i_units IN ('Bunch', 'Gram'),i_size IN ('economy', 'petite')]]]] and i_category IN ('Men', 'Women'))
 --------------------------------PhysicalOlapScan[item]
 

From 60b490313b402faee1e66863da708ec069a70292 Mon Sep 17 00:00:00 2001
From: feiniaofeiafei <moailing@selectdb.com>
Date: Thu, 9 Jan 2025 16:16:11 +0800
Subject: [PATCH 076/140] [fix](nereids) adjust logical repeat property derive
 (#44360)

### What problem does this PR solve?

Problem Summary:
Before this pr, if the children of PhysicalRepeat is hash-distributed
based on a and b, and the common columns that can be extracted from the
PhysicalRepeat grouping sets are a, b, and c, the PhysicalRepeat will
output Any distribution property.
This pr allow PhysicalRepeat maintain child hash distribution property
in this situaltion.
---
 .../ChildOutputPropertyDeriver.java           | 24 +++++++------------
 .../ChildOutputPropertyDeriverTest.java       | 23 ++++++++++++++++++
 2 files changed, 32 insertions(+), 15 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
index 8f191b61286e431..8dc77a43f50d4d3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
@@ -71,6 +71,7 @@
 import com.google.common.collect.Sets;
 
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -372,22 +373,15 @@ public PhysicalProperties visitPhysicalRepeat(PhysicalRepeat<? extends Plan> rep
                     );
                 }
                 List<ExprId> orderedShuffledColumns = distributionSpecHash.getOrderedShuffledColumns();
-                if (!intersectGroupingKeys.isEmpty() && intersectGroupingKeys.size()
-                        >= Sets.newHashSet(orderedShuffledColumns).size()) {
-                    boolean hashColumnsChanged = false;
-                    for (Expression intersectGroupingKey : intersectGroupingKeys) {
-                        if (!(intersectGroupingKey instanceof SlotReference)) {
-                            hashColumnsChanged = true;
-                            break;
-                        }
-                        if (!(orderedShuffledColumns.contains(((SlotReference) intersectGroupingKey).getExprId()))) {
-                            hashColumnsChanged = true;
-                            break;
-                        }
-                    }
-                    if (!hashColumnsChanged) {
-                        return childrenOutputProperties.get(0);
+                Set<ExprId> intersectGroupingKeysId = new HashSet<>();
+                for (Expression key : intersectGroupingKeys) {
+                    if (!(key instanceof SlotReference)) {
+                        break;
                     }
+                    intersectGroupingKeysId.add(((SlotReference) key).getExprId());
+                }
+                if (intersectGroupingKeysId.containsAll(orderedShuffledColumns)) {
+                    return childrenOutputProperties.get(0);
                 }
             }
             output = PhysicalProperties.createAnyFromHash((DistributionSpecHash) childDistributionSpec);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriverTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriverTest.java
index a39bc664e1c1de9..8351b9f8c2263a0 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriverTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriverTest.java
@@ -885,4 +885,27 @@ void testRepeatReturnChild() {
         PhysicalProperties result = deriver.getOutputProperties(null, groupExpression);
         Assertions.assertEquals(child, result);
     }
+
+    @Test
+    void testRepeatReturnChild2() {
+        SlotReference c1 = new SlotReference(
+                new ExprId(1), "c1", TinyIntType.INSTANCE, true, ImmutableList.of());
+        SlotReference c2 = new SlotReference(
+                new ExprId(2), "c2", TinyIntType.INSTANCE, true, ImmutableList.of());
+        SlotReference c3 = new SlotReference(
+                new ExprId(3), "c3", TinyIntType.INSTANCE, true, ImmutableList.of());
+        PhysicalRepeat<GroupPlan> repeat = new PhysicalRepeat<>(
+                ImmutableList.of(ImmutableList.of(c1, c2, c3), ImmutableList.of(c1, c2), ImmutableList.of(c1, c2)),
+                ImmutableList.of(c1, c2, c3),
+                logicalProperties,
+                groupPlan
+        );
+        GroupExpression groupExpression = new GroupExpression(repeat);
+        new Group(null, groupExpression, null);
+        PhysicalProperties child = PhysicalProperties.createHash(
+                ImmutableList.of(new ExprId(1)), ShuffleType.EXECUTION_BUCKETED);
+        ChildOutputPropertyDeriver deriver = new ChildOutputPropertyDeriver(Lists.newArrayList(child));
+        PhysicalProperties result = deriver.getOutputProperties(null, groupExpression);
+        Assertions.assertEquals(child, result);
+    }
 }

From 43bf9e1cd1ba25cdcdb7afd6b42464e8c8b65173 Mon Sep 17 00:00:00 2001
From: zgxme <zhenggaoxiong@selectdb.com>
Date: Thu, 9 Jan 2025 16:30:15 +0800
Subject: [PATCH 077/140] [chore](regression) Normalize external case labels
 (#46622)

### What problem does this PR solve?
Ensure that all external cases are labeled with the external tag.
---
 .../suites/external_table_p0/jdbc/test_jdbc_query_tvf.groovy    | 2 +-
 .../jdbc/test_switch_catalog_and_delete_internal.groovy         | 2 +-
 .../external_table_p0/tvf/test_frontends_disks_tvf.groovy       | 2 +-
 .../tvf/test_insert_from_tvf_with_common_user.groovy            | 2 +-
 .../external_table_p0/tvf/test_local_tvf_compression.groovy     | 2 +-
 .../suites/external_table_p0/tvf/test_local_tvf_enclose.groovy  | 2 +-
 .../tvf/test_local_tvf_parquet_unsigned_integers.groovy         | 2 +-
 .../tvf/test_local_tvf_with_complex_type.groovy                 | 2 +-
 .../tvf/test_local_tvf_with_complex_type_element_at.groovy      | 2 +-
 .../test_local_tvf_with_complex_type_insertinto_doris.groovy    | 2 +-
 .../tvf/test_read_csv_empty_line_as_null.groovy                 | 2 +-
 regression-test/suites/external_table_p0/tvf/test_s3_tvf.groovy | 2 +-
 .../suites/external_table_p0/tvf/test_s3_tvf_compression.groovy | 2 +-
 .../external_table_p0/tvf/test_s3_tvf_parquet_compress.groovy   | 2 +-
 .../external_table_p0/tvf/test_s3_tvf_with_resource.groovy      | 2 +-
 .../suites/external_table_p0/tvf/test_tvf_csv_line_end.groovy   | 2 +-
 16 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/regression-test/suites/external_table_p0/jdbc/test_jdbc_query_tvf.groovy b/regression-test/suites/external_table_p0/jdbc/test_jdbc_query_tvf.groovy
index 0c1e04afda9cf4a..bee77ab73f073c4 100644
--- a/regression-test/suites/external_table_p0/jdbc/test_jdbc_query_tvf.groovy
+++ b/regression-test/suites/external_table_p0/jdbc/test_jdbc_query_tvf.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_jdbc_query_tvf") {
+suite("test_jdbc_query_tvf", "p0,external") {
 
     String enabled = context.config.otherConfigs.get("enableJdbcTest")
     String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
diff --git a/regression-test/suites/external_table_p0/jdbc/test_switch_catalog_and_delete_internal.groovy b/regression-test/suites/external_table_p0/jdbc/test_switch_catalog_and_delete_internal.groovy
index 1e9e2ffdf48cbb5..adfb20e355d258b 100644
--- a/regression-test/suites/external_table_p0/jdbc/test_switch_catalog_and_delete_internal.groovy
+++ b/regression-test/suites/external_table_p0/jdbc/test_switch_catalog_and_delete_internal.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_switch_catalog_and_delete_internal") {
+suite("test_switch_catalog_and_delete_internal", "p0,external") {
     String enabled = context.config.otherConfigs.get("enableJdbcTest")
     String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
     String mysql_port = context.config.otherConfigs.get("mysql_57_port");
diff --git a/regression-test/suites/external_table_p0/tvf/test_frontends_disks_tvf.groovy b/regression-test/suites/external_table_p0/tvf/test_frontends_disks_tvf.groovy
index 0cb282539080ffc..b79c3e046984cc8 100644
--- a/regression-test/suites/external_table_p0/tvf/test_frontends_disks_tvf.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_frontends_disks_tvf.groovy
@@ -16,7 +16,7 @@
 // under the License.
 
 // This suit test the `frontends_disks` tvf
-suite("test_frontends_disks_tvf") {
+suite("test_frontends_disks_tvf", "p0,external") {
     List<List<Object>> table =  sql """ select * from `frontends_disks`(); """
     assertTrue(table.size() > 0)
     assertTrue(table[0].size == 10)
diff --git a/regression-test/suites/external_table_p0/tvf/test_insert_from_tvf_with_common_user.groovy b/regression-test/suites/external_table_p0/tvf/test_insert_from_tvf_with_common_user.groovy
index c23878b4d1b64f0..91e3ab915ebe207 100644
--- a/regression-test/suites/external_table_p0/tvf/test_insert_from_tvf_with_common_user.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_insert_from_tvf_with_common_user.groovy
@@ -21,7 +21,7 @@ import java.nio.charset.StandardCharsets
 import java.nio.file.Files
 import java.nio.file.Paths
 
-suite("test_insert_from_tvf_with_common_user", "p0") {
+suite("test_insert_from_tvf_with_common_user", "p0,external") {
     String ak = getS3AK()
     String sk = getS3SK()
     String s3_endpoint = getS3Endpoint()
diff --git a/regression-test/suites/external_table_p0/tvf/test_local_tvf_compression.groovy b/regression-test/suites/external_table_p0/tvf/test_local_tvf_compression.groovy
index 420887fb41d6b66..3b08001f96f25fd 100644
--- a/regression-test/suites/external_table_p0/tvf/test_local_tvf_compression.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_local_tvf_compression.groovy
@@ -17,7 +17,7 @@ import org.junit.Assert
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_local_tvf_compression", "p0,tvf") {
+suite("test_local_tvf_compression", "p0,tvf,external") {
     List<List<Object>> backends =  sql """ show backends """
     assertTrue(backends.size() > 0)
     def be_id = backends[0][0]
diff --git a/regression-test/suites/external_table_p0/tvf/test_local_tvf_enclose.groovy b/regression-test/suites/external_table_p0/tvf/test_local_tvf_enclose.groovy
index e7437cd20ec13ef..28e000d68ddd195 100644
--- a/regression-test/suites/external_table_p0/tvf/test_local_tvf_enclose.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_local_tvf_enclose.groovy
@@ -17,7 +17,7 @@ import org.junit.Assert
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_local_tvf_enclose", "p0,tvf") {
+suite("test_local_tvf_enclose", "p0,tvf,external") {
     List<List<Object>> backends =  sql """ show backends """
     assertTrue(backends.size() > 0)
     def be_id = backends[0][0]
diff --git a/regression-test/suites/external_table_p0/tvf/test_local_tvf_parquet_unsigned_integers.groovy b/regression-test/suites/external_table_p0/tvf/test_local_tvf_parquet_unsigned_integers.groovy
index 24cfb5f2ac207cf..aeb032e941bed52 100644
--- a/regression-test/suites/external_table_p0/tvf/test_local_tvf_parquet_unsigned_integers.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_local_tvf_parquet_unsigned_integers.groovy
@@ -18,7 +18,7 @@ import org.junit.Assert
 // under the License.
 
 // This suit test the `backends` tvf
-suite("test_local_tvf_parquet_unsigned_integers", "p0") {
+suite("test_local_tvf_parquet_unsigned_integers", "p0,external") {
     List<List<Object>> backends =  sql """ show backends """
     def dataFilePath = context.config.dataPath + "/external_table_p0/tvf/"
 
diff --git a/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type.groovy b/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type.groovy
index cd5948b22c36fd9..a990a8873fa29eb 100644
--- a/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type.groovy
@@ -18,7 +18,7 @@ import org.junit.Assert
 // under the License.
 
 // This suit test the `backends` tvf
-suite("test_local_tvf_with_complex_type", "p0") {
+suite("test_local_tvf_with_complex_type", "p0,external") {
     List<List<Object>> backends =  sql """ show backends """
     def dataFilePath = context.config.dataPath + "/external_table_p0/tvf/"
 
diff --git a/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_element_at.groovy b/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_element_at.groovy
index 91c00a9046affc3..28ddf9c7a51fe58 100644
--- a/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_element_at.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_element_at.groovy
@@ -18,7 +18,7 @@ import org.junit.Assert
 // under the License.
 
 // This suit test the `backends` tvf
-suite("test_local_tvf_with_complex_type_element_at", "p0") {
+suite("test_local_tvf_with_complex_type_element_at", "p0,external") {
     List<List<Object>> backends =  sql """ show backends """
     assertTrue(backends.size() > 0)
     def be_id = backends[0][0]
diff --git a/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy b/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy
index 596ed79eec9dc2f..188d7069ae72e1e 100644
--- a/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy
@@ -18,7 +18,7 @@ import org.junit.Assert
 // under the License.
 
 // This suit test the `backends` tvf
-suite("test_local_tvf_with_complex_type_insertinto_doris", "p0") {
+suite("test_local_tvf_with_complex_type_insertinto_doris", "p0,external") {
     List<List<Object>> backends =  sql """ select * from backends(); """
     assertTrue(backends.size() > 0)
     def be_id = backends[0][0]
diff --git a/regression-test/suites/external_table_p0/tvf/test_read_csv_empty_line_as_null.groovy b/regression-test/suites/external_table_p0/tvf/test_read_csv_empty_line_as_null.groovy
index 0a64109eb228f9d..b2de1e5270c2d67 100644
--- a/regression-test/suites/external_table_p0/tvf/test_read_csv_empty_line_as_null.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_read_csv_empty_line_as_null.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_read_csv_empty_line_as_null", "p0") {
+suite("test_read_csv_empty_line_as_null", "p0,external") {
     // open nereids
     sql """ set enable_nereids_planner=true """
     sql """ set enable_fallback_to_original_planner=false """
diff --git a/regression-test/suites/external_table_p0/tvf/test_s3_tvf.groovy b/regression-test/suites/external_table_p0/tvf/test_s3_tvf.groovy
index 67dc94d32d0c127..8ab9793c5d31c72 100644
--- a/regression-test/suites/external_table_p0/tvf/test_s3_tvf.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_s3_tvf.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_s3_tvf", "p0") {
+suite("test_s3_tvf", "p0,external") {
     // open nereids
     sql """ set enable_nereids_planner=true """
     sql """ set enable_fallback_to_original_planner=false """
diff --git a/regression-test/suites/external_table_p0/tvf/test_s3_tvf_compression.groovy b/regression-test/suites/external_table_p0/tvf/test_s3_tvf_compression.groovy
index 2975f4f0cc9bff7..6e44cd82db90439 100644
--- a/regression-test/suites/external_table_p0/tvf/test_s3_tvf_compression.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_s3_tvf_compression.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_s3_tvf_compression", "p0") {
+suite("test_s3_tvf_compression", "p0,external") {
     
     String ak = getS3AK()
     String sk = getS3SK()
diff --git a/regression-test/suites/external_table_p0/tvf/test_s3_tvf_parquet_compress.groovy b/regression-test/suites/external_table_p0/tvf/test_s3_tvf_parquet_compress.groovy
index e8ca08811f018e6..76c773e0a44a276 100644
--- a/regression-test/suites/external_table_p0/tvf/test_s3_tvf_parquet_compress.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_s3_tvf_parquet_compress.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_s3_tvf_parquet_compress", "p0") {
+suite("test_s3_tvf_parquet_compress", "p0,external") {
 //parquet data page v2 test 
 
 
diff --git a/regression-test/suites/external_table_p0/tvf/test_s3_tvf_with_resource.groovy b/regression-test/suites/external_table_p0/tvf/test_s3_tvf_with_resource.groovy
index bbee5fccc4ca07c..1d243e9dd2f27ad 100644
--- a/regression-test/suites/external_table_p0/tvf/test_s3_tvf_with_resource.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_s3_tvf_with_resource.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_s3_tvf_with_resource", "p0") {
+suite("test_s3_tvf_with_resource", "p0,external") {
     // open nereids
     sql """ set enable_nereids_planner=true """
     sql """ set enable_fallback_to_original_planner=false """
diff --git a/regression-test/suites/external_table_p0/tvf/test_tvf_csv_line_end.groovy b/regression-test/suites/external_table_p0/tvf/test_tvf_csv_line_end.groovy
index cb2beb6f941479d..3b08cede4074871 100644
--- a/regression-test/suites/external_table_p0/tvf/test_tvf_csv_line_end.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_tvf_csv_line_end.groovy
@@ -17,7 +17,7 @@ import org.junit.Assert
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_tvf_csv_line_end", "p0,tvf") {
+suite("test_tvf_csv_line_end", "p0,tvf,external") {
     List<List<Object>> backends =  sql """ show backends """
     assertTrue(backends.size() > 0)
     def be_id = backends[0][0]

From c206a6bbb146447ba453e26a537996aa5f808d7e Mon Sep 17 00:00:00 2001
From: zhangdong <zhangdong@selectdb.com>
Date: Thu, 9 Jan 2025 16:32:25 +0800
Subject: [PATCH 078/140] [enhance](mtmv)When obtaining the partition list
 fails, treat the paimon table as an unpartitioned table  (#46641)

### What problem does this PR solve?

When retrieving data of type Paimon Date in version 0.9 from the system
table, the value is an integer and cannot be converted to type Date.

This issue has been fixed in Paimon's latest code.

This PR downgrades this situation without affecting user data queries
---
 .../paimon/run01.sql                          | 14 ++++++++-
 .../paimon/PaimonExternalTable.java           | 11 +++++++
 .../paimon/PaimonPartitionInfo.java           |  5 ++++
 .../doris/datasource/paimon/PaimonUtil.java   | 11 +++++--
 .../data/mtmv_p0/test_paimon_mtmv.out         |  6 ++++
 .../suites/mtmv_p0/test_paimon_mtmv.groovy    | 30 +++++++++++++++++++
 6 files changed, 74 insertions(+), 3 deletions(-)

diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run01.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run01.sql
index 5cc0a0ea685e37f..7722d09463686e1 100644
--- a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run01.sql
+++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run01.sql
@@ -55,4 +55,16 @@ insert into null_partition values(1,'bj');
 insert into null_partition values(2,null);
 insert into null_partition values(3,NULL);
 insert into null_partition values(4,'null');
-insert into null_partition values(5,'NULL');
\ No newline at end of file
+insert into null_partition values(5,'NULL');
+
+drop table if exists date_partition;
+CREATE TABLE date_partition (
+                               id BIGINT,
+                               create_date DATE
+) PARTITIONED BY (create_date) TBLPROPERTIES (
+    'primary-key' = 'create_date,id',
+    'bucket'=10,
+    'file.format'='orc'
+);
+
+insert into date_partition values(1,date '2020-01-01');
\ No newline at end of file
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
index 7264d83621d8beb..347e8df5c8f75e1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
@@ -165,6 +165,9 @@ public Map<String, PartitionItem> getAndCopyPartitionItems(Optional<MvccSnapshot
 
     @Override
     public PartitionType getPartitionType(Optional<MvccSnapshot> snapshot) {
+        if (isPartitionInvalid(snapshot)) {
+            return PartitionType.UNPARTITIONED;
+        }
         return getPartitionColumns(snapshot).size() > 0 ? PartitionType.LIST : PartitionType.UNPARTITIONED;
     }
 
@@ -176,9 +179,17 @@ public Set<String> getPartitionColumnNames(Optional<MvccSnapshot> snapshot) {
 
     @Override
     public List<Column> getPartitionColumns(Optional<MvccSnapshot> snapshot) {
+        if (isPartitionInvalid(snapshot)) {
+            return Collections.emptyList();
+        }
         return getPaimonSchemaCacheValue(snapshot).getPartitionColumns();
     }
 
+    private boolean isPartitionInvalid(Optional<MvccSnapshot> snapshot) {
+        PaimonSnapshotCacheValue paimonSnapshotCacheValue = getOrFetchSnapshotCacheValue(snapshot);
+        return paimonSnapshotCacheValue.getPartitionInfo().isPartitionInvalid();
+    }
+
     @Override
     public MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshContext context,
             Optional<MvccSnapshot> snapshot)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java
index 4d3326f8e483764..88515a2510d2c72 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java
@@ -45,4 +45,9 @@ public Map<String, PartitionItem> getNameToPartitionItem() {
     public Map<String, PaimonPartition> getNameToPartition() {
         return nameToPartition;
     }
+
+    public boolean isPartitionInvalid() {
+        // when transfer to partitionItem failed, will not equal
+        return nameToPartitionItem.size() != nameToPartition.size();
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
index 1f7576dca51d934..b3df41bc5cef175 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
@@ -117,7 +117,7 @@ public static PaimonPartition rowToPartition(InternalRow row) {
     }
 
     public static PaimonPartitionInfo generatePartitionInfo(List<Column> partitionColumns,
-            List<PaimonPartition> paimonPartitions) throws AnalysisException {
+            List<PaimonPartition> paimonPartitions) {
         Map<String, PartitionItem> nameToPartitionItem = Maps.newHashMap();
         Map<String, PaimonPartition> nameToPartition = Maps.newHashMap();
         PaimonPartitionInfo partitionInfo = new PaimonPartitionInfo(nameToPartitionItem, nameToPartition);
@@ -127,7 +127,14 @@ public static PaimonPartitionInfo generatePartitionInfo(List<Column> partitionCo
         for (PaimonPartition paimonPartition : paimonPartitions) {
             String partitionName = getPartitionName(partitionColumns, paimonPartition.getPartitionValues());
             nameToPartition.put(partitionName, paimonPartition);
-            nameToPartitionItem.put(partitionName, toListPartitionItem(partitionName, partitionColumns));
+            try {
+                // partition values return by paimon api, may have problem,
+                // to avoid affecting the query, we catch exceptions here
+                nameToPartitionItem.put(partitionName, toListPartitionItem(partitionName, partitionColumns));
+            } catch (Exception e) {
+                LOG.warn("toListPartitionItem failed, partitionColumns: {}, partitionValues: {}", partitionColumns,
+                        paimonPartition.getPartitionValues(), e);
+            }
         }
         return partitionInfo;
     }
diff --git a/regression-test/data/mtmv_p0/test_paimon_mtmv.out b/regression-test/data/mtmv_p0/test_paimon_mtmv.out
index ba6fc06c1d24917..0cfe4bd293cf51d 100644
--- a/regression-test/data/mtmv_p0/test_paimon_mtmv.out
+++ b/regression-test/data/mtmv_p0/test_paimon_mtmv.out
@@ -140,3 +140,9 @@ true
 4	null
 5	NULL
 
+-- !date_partition_base_table --
+1	2020-01-01
+
+-- !date_partition --
+1	2020-01-01
+
diff --git a/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy b/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy
index 48d63e03ec3db52..98477ab92d542e9 100644
--- a/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy
@@ -264,6 +264,36 @@ suite("test_paimon_mtmv", "p0,external,mtmv,external_docker,external_docker_dori
     order_qt_null_partition "SELECT * FROM ${mvName} "
     sql """drop materialized view if exists ${mvName};"""
 
+    // date type will has problem
+    order_qt_date_partition_base_table "SELECT * FROM ${catalogName}.`test_paimon_spark`.date_partition"
+    test {
+         sql """
+            CREATE MATERIALIZED VIEW ${mvName}
+                BUILD DEFERRED REFRESH AUTO ON MANUAL
+                partition by (`create_date`)
+                DISTRIBUTED BY RANDOM BUCKETS 2
+                PROPERTIES ('replication_num' = '1')
+                AS
+                SELECT * FROM ${catalogName}.`test_paimon_spark`.date_partition;
+            """
+          exception "Unable to find a suitable base table"
+      }
+
+    sql """
+        CREATE MATERIALIZED VIEW ${mvName}
+            BUILD DEFERRED REFRESH AUTO ON MANUAL
+            DISTRIBUTED BY RANDOM BUCKETS 2
+            PROPERTIES ('replication_num' = '1')
+            AS
+            SELECT * FROM ${catalogName}.`test_paimon_spark`.date_partition;
+        """
+    sql """
+         REFRESH MATERIALIZED VIEW ${mvName} auto;
+     """
+    waitingMTMVTaskFinishedByMvName(mvName)
+    order_qt_date_partition "SELECT * FROM ${mvName} "
+
+    sql """drop materialized view if exists ${mvName};"""
     sql """drop catalog if exists ${catalogName}"""
 
 }

From 1258ae31431c01056ca76cba3b86fe9c35d2e62d Mon Sep 17 00:00:00 2001
From: zzzxl <yangsiyu@selectdb.com>
Date: Thu, 9 Jan 2025 16:58:49 +0800
Subject: [PATCH 079/140] [fix](test) fix for test_build_index_fault case
 (#46692)

---
 .../suites/fault_injection_p0/test_build_index_fault.groovy   | 4 ++--
 .../test_build_index_exception_fault_injection.groovy         | 1 -
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/regression-test/suites/fault_injection_p0/test_build_index_fault.groovy b/regression-test/suites/fault_injection_p0/test_build_index_fault.groovy
index 3a33c2e41ecf57e..27914aceb4a8d3e 100644
--- a/regression-test/suites/fault_injection_p0/test_build_index_fault.groovy
+++ b/regression-test/suites/fault_injection_p0/test_build_index_fault.groovy
@@ -234,14 +234,14 @@ suite("test_build_index_fault", "inverted_index, nonConcurrent,p2"){
 
     // BUILD INDEX with error injection
     sql """ ALTER TABLE ${tableName} ADD INDEX idx_url (`url`) USING INVERTED """
-    GetDebugPoint().enableDebugPointForAllBEs("IndexBuilder::handle_single_rowset")
+    GetDebugPoint().enableDebugPointForAllBEs("IndexBuilder::handle_single_rowset_iterator_next_batch_error")
     sql """ BUILD INDEX idx_url ON ${tableName}; """
     state = wait_for_last_build_index_on_table_finish(tableName, timeout)
     assertEquals("CANCELLED", state)
     // check data
     qt_count7 """ SELECT COUNT() from ${tableName}; """
 
-    GetDebugPoint().disableDebugPointForAllBEs("IndexBuilder::handle_single_rowset")
+    GetDebugPoint().disableDebugPointForAllBEs("IndexBuilder::handle_single_rowset_iterator_next_batch_error")
     // rebuild index
     sql """ BUILD INDEX idx_url ON ${tableName}; """
     state = wait_for_last_build_index_on_table_finish(tableName, timeout)
diff --git a/regression-test/suites/fault_injection_p2/test_build_index_exception_fault_injection.groovy b/regression-test/suites/fault_injection_p2/test_build_index_exception_fault_injection.groovy
index 94cf4d6c15c1dae..14e74d9b8eb2a26 100644
--- a/regression-test/suites/fault_injection_p2/test_build_index_exception_fault_injection.groovy
+++ b/regression-test/suites/fault_injection_p2/test_build_index_exception_fault_injection.groovy
@@ -221,7 +221,6 @@ suite("test_build_index_exception_fault_injection", "nonConcurrent,p2") {
         "IndexBuilder::handle_single_rowset_support_inverted_index",
         "IndexBuilder::handle_single_rowset_index_column_writer_create_error",
         "IndexBuilder::handle_single_rowset_create_iterator_error",
-        "IndexBuilder::handle_single_rowset",
         "IndexBuilder::handle_single_rowset_iterator_next_batch_error",
         "IndexBuilder::handle_single_rowset_write_inverted_index_data_error",
         "IndexBuilder::handle_single_rowset_index_build_finish_error",

From 394caaf6850ca989fa09384abf47c4c60191a920 Mon Sep 17 00:00:00 2001
From: shee <13843187+qzsee@users.noreply.github.com>
Date: Thu, 9 Jan 2025 17:14:04 +0800
Subject: [PATCH 080/140] [Feature](nereids) support median agg function using
 percentile (#46377)

### What problem does this PR solve?

reference :
https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/median

---------

Co-authored-by: garenshi <garenshi@tencent.com>
---
 .../catalog/BuiltinAggregateFunctions.java    |   2 +
 .../expression/ExpressionNormalization.java   |   2 +
 .../rules/expression/ExpressionRuleType.java  |   1 +
 .../rules/expression/rules/MedianConvert.java |  46 ++++++++
 .../expressions/functions/agg/Median.java     | 106 ++++++++++++++++++
 .../visitor/AggregateFunctionVisitor.java     |   5 +
 .../test_convert_median_to_percentile.out     |  21 ++++
 .../test_convert_median_to_percentile.groovy  |  78 +++++++++++++
 8 files changed, 261 insertions(+)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/MedianConvert.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Median.java
 create mode 100644 regression-test/data/nereids_rules_p0/expression/test_convert_median_to_percentile.out
 create mode 100644 regression-test/suites/nereids_rules_p0/expression/test_convert_median_to_percentile.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinAggregateFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinAggregateFunctions.java
index 655a3dfce2985d4..5e2e72919874701 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinAggregateFunctions.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinAggregateFunctions.java
@@ -50,6 +50,7 @@
 import org.apache.doris.nereids.trees.expressions.functions.agg.MapAgg;
 import org.apache.doris.nereids.trees.expressions.functions.agg.Max;
 import org.apache.doris.nereids.trees.expressions.functions.agg.MaxBy;
+import org.apache.doris.nereids.trees.expressions.functions.agg.Median;
 import org.apache.doris.nereids.trees.expressions.functions.agg.Min;
 import org.apache.doris.nereids.trees.expressions.functions.agg.MinBy;
 import org.apache.doris.nereids.trees.expressions.functions.agg.MultiDistinctCount;
@@ -131,6 +132,7 @@ public class BuiltinAggregateFunctions implements FunctionHelper {
             agg(MapAgg.class, "map_agg"),
             agg(Max.class, "max"),
             agg(MaxBy.class, "max_by"),
+            agg(Median.class, "median"),
             agg(Min.class, "min"),
             agg(MinBy.class, "min_by"),
             agg(MultiDistinctCount.class, "multi_distinct_count"),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java
index 0e52f2aaaf65271..b4430d33087e917 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java
@@ -23,6 +23,7 @@
 import org.apache.doris.nereids.rules.expression.rules.FoldConstantRule;
 import org.apache.doris.nereids.rules.expression.rules.InPredicateDedup;
 import org.apache.doris.nereids.rules.expression.rules.InPredicateToEqualToRule;
+import org.apache.doris.nereids.rules.expression.rules.MedianConvert;
 import org.apache.doris.nereids.rules.expression.rules.MergeDateTrunc;
 import org.apache.doris.nereids.rules.expression.rules.NormalizeBinaryPredicatesRule;
 import org.apache.doris.nereids.rules.expression.rules.SimplifyArithmeticComparisonRule;
@@ -52,6 +53,7 @@ public class ExpressionNormalization extends ExpressionRewrite {
                 FoldConstantRule.INSTANCE,
                 SimplifyCastRule.INSTANCE,
                 DigitalMaskingConvert.INSTANCE,
+                MedianConvert.INSTANCE,
                 SimplifyArithmeticComparisonRule.INSTANCE,
                 ConvertAggStateCast.INSTANCE,
                 MergeDateTrunc.INSTANCE,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java
index 16881a61addb234..bc12c0459eefb5d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java
@@ -38,6 +38,7 @@ public enum ExpressionRuleType {
     IN_PREDICATE_TO_EQUAL_TO,
     LIKE_TO_EQUAL,
     MERGE_DATE_TRUNC,
+    MEDIAN_CONVERT,
     NORMALIZE_BINARY_PREDICATES,
     NULL_SAFE_EQUAL_TO_EQUAL,
     REPLACE_VARIABLE_BY_LITERAL,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/MedianConvert.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/MedianConvert.java
new file mode 100644
index 000000000000000..e6e295ef98c2a85
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/MedianConvert.java
@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.expression.rules;
+
+import org.apache.doris.nereids.rules.expression.ExpressionPatternMatcher;
+import org.apache.doris.nereids.rules.expression.ExpressionPatternRuleFactory;
+import org.apache.doris.nereids.rules.expression.ExpressionRuleType;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.agg.Median;
+import org.apache.doris.nereids.trees.expressions.functions.agg.Percentile;
+import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/**
+ * median(col) -> percentile(col, 0.5)
+ */
+public class MedianConvert implements ExpressionPatternRuleFactory {
+    public static MedianConvert INSTANCE = new MedianConvert();
+
+    @Override
+    public List<ExpressionPatternMatcher<? extends Expression>> buildRules() {
+        return ImmutableList.of(
+                matchesType(Median.class).then(median ->
+                    new Percentile(median.child(0), DoubleLiteral.of(0.5))
+                ).toRule(ExpressionRuleType.MEDIAN_CONVERT)
+        );
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Median.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Median.java
new file mode 100644
index 000000000000000..342604f526c3d4f
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Median.java
@@ -0,0 +1,106 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions.agg;
+
+import org.apache.doris.catalog.FunctionSignature;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
+import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.types.BigIntType;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.DoubleType;
+import org.apache.doris.nereids.types.FloatType;
+import org.apache.doris.nereids.types.IntegerType;
+import org.apache.doris.nereids.types.LargeIntType;
+import org.apache.doris.nereids.types.SmallIntType;
+import org.apache.doris.nereids.types.TinyIntType;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/**
+ * AggregateFunction 'median'. This class is generated by GenerateFunction.
+ */
+public class Median extends NullableAggregateFunction
+        implements UnaryExpression, ExplicitlyCastableSignature {
+
+    public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
+            FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE),
+            FunctionSignature.ret(DoubleType.INSTANCE).args(FloatType.INSTANCE),
+            FunctionSignature.ret(DoubleType.INSTANCE).args(LargeIntType.INSTANCE),
+            FunctionSignature.ret(DoubleType.INSTANCE).args(BigIntType.INSTANCE),
+            FunctionSignature.ret(DoubleType.INSTANCE).args(IntegerType.INSTANCE),
+            FunctionSignature.ret(DoubleType.INSTANCE).args(SmallIntType.INSTANCE),
+            FunctionSignature.ret(DoubleType.INSTANCE).args(TinyIntType.INSTANCE)
+
+    );
+
+    /**
+     * constructor with 2 arguments.
+     */
+    public Median(Expression arg) {
+        this(false, arg);
+    }
+
+    /**
+     * constructor with 2 arguments.
+     */
+    public Median(boolean distinct, Expression arg) {
+        this(distinct, false, arg);
+    }
+
+    public Median(boolean distinct, boolean alwaysNullable, Expression arg) {
+        super("median", distinct, alwaysNullable, arg);
+    }
+
+    @Override
+    public void checkLegalityBeforeTypeCoercion() {
+        DataType argType = child().getDataType();
+        if (((!argType.isNumericType() && !argType.isNullType()) || argType.isOnlyMetricType())) {
+            throw new AnalysisException("median requires a numeric parameter: " + toSql());
+        }
+    }
+
+    /**
+     * withDistinctAndChildren.
+     */
+    @Override
+    public Median withDistinctAndChildren(boolean distinct, List<Expression> children) {
+        Preconditions.checkArgument(children.size() == 1);
+        return new Median(distinct, alwaysNullable, children.get(0));
+    }
+
+    @Override
+    public NullableAggregateFunction withAlwaysNullable(boolean alwaysNullable) {
+        return new Median(distinct, alwaysNullable, children.get(0));
+    }
+
+    @Override
+    public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
+        return visitor.visitMedian(this, context);
+    }
+
+    @Override
+    public List<FunctionSignature> getSignatures() {
+        return SIGNATURES;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/AggregateFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/AggregateFunctionVisitor.java
index 50ca233cdbc703c..1918eb1b7b43136 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/AggregateFunctionVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/AggregateFunctionVisitor.java
@@ -51,6 +51,7 @@
 import org.apache.doris.nereids.trees.expressions.functions.agg.MapAgg;
 import org.apache.doris.nereids.trees.expressions.functions.agg.Max;
 import org.apache.doris.nereids.trees.expressions.functions.agg.MaxBy;
+import org.apache.doris.nereids.trees.expressions.functions.agg.Median;
 import org.apache.doris.nereids.trees.expressions.functions.agg.Min;
 import org.apache.doris.nereids.trees.expressions.functions.agg.MinBy;
 import org.apache.doris.nereids.trees.expressions.functions.agg.MultiDistinctCount;
@@ -261,6 +262,10 @@ default R visitOrthogonalBitmapUnionCount(OrthogonalBitmapUnionCount function, C
         return visitAggregateFunction(function, context);
     }
 
+    default R visitMedian(Median median, C context) {
+        return visitNullableAggregateFunction(median, context);
+    }
+
     default R visitPercentile(Percentile percentile, C context) {
         return visitNullableAggregateFunction(percentile, context);
     }
diff --git a/regression-test/data/nereids_rules_p0/expression/test_convert_median_to_percentile.out b/regression-test/data/nereids_rules_p0/expression/test_convert_median_to_percentile.out
new file mode 100644
index 000000000000000..3cccc3d6fb086ba
--- /dev/null
+++ b/regression-test/data/nereids_rules_p0/expression/test_convert_median_to_percentile.out
@@ -0,0 +1,21 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_1 --
+101
+
+-- !select_2 --
+101
+
+-- !select_3 --
+2000	100
+2001	125.5
+
+-- !select_4 --
+2000	100
+2001	125.5
+
+-- !select_5 --
+2001	125.5
+
+-- !select_6 --
+2001	125.5
+
diff --git a/regression-test/suites/nereids_rules_p0/expression/test_convert_median_to_percentile.groovy b/regression-test/suites/nereids_rules_p0/expression/test_convert_median_to_percentile.groovy
new file mode 100644
index 000000000000000..dd5b8032df655f6
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/expression/test_convert_median_to_percentile.groovy
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_convert_median_to_percentile") {
+    sql "SET enable_nereids_planner=true"
+    sql "SET enable_fallback_to_original_planner=false"
+
+    sql "create database if not exists test_convert_median_to_percentile"
+    sql "use test_convert_median_to_percentile"
+
+    sql "DROP TABLE IF EXISTS sales"
+    sql """
+           CREATE TABLE sales (
+               year INT,
+               country STRING,
+               product STRING,
+               profit INT
+            ) 
+            DISTRIBUTED BY HASH(`year`) BUCKETS 1
+            PROPERTIES (
+            "replication_num" = "1"
+            )
+        """
+    sql """
+        INSERT INTO sales VALUES
+        (2000,'Finland','Computer',1501),
+        (2000,'Finland','Phone',100),
+        (2001,'Finland','Phone',10),
+        (2000,'India','Calculator',75),
+        (2000,'India','Calculator',76),
+        (2000,'India','Computer',1201),
+        (2000,'USA','Calculator',77),
+        (2000,'USA','Computer',1502),
+        (2001,'USA','Calculator',50),
+        (2001,'USA','Computer',1503),
+        (2001,'USA','Computer',1202),
+        (2001,'USA','TV',150),
+        (2001,'USA','TV',101);
+        """
+
+    def sql1 = "select median(profit) from sales"
+    def sql2 = "select percentile(profit, 0.5) from sales"
+    def explainStr1 = sql """ explain  ${sql1} """
+    assertTrue(explainStr1.toString().contains("percentile(profit, 0.5)"))
+    qt_select_1 "${sql1}"
+    qt_select_2 "${sql2}"
+
+    def sql3 = "select year, median(profit) from sales group by year order by year"
+    def sql4 = "select year, percentile(profit, 0.5) from sales group by year order by year"
+    def explainStr3 = sql """ explain  ${sql3} """
+    assertTrue(explainStr3.toString().contains("percentile(profit, 0.5)"))
+    qt_select_3 "${sql3}"
+    qt_select_4 "${sql4}"
+
+    def sql5 = "select year, median(profit) from sales group by year having median(profit) > 100"
+    def sql6 = "select year, percentile(profit, 0.5) from sales group by year having percentile(profit, 0.5) > 100"
+    def explainStr5 = sql """ explain  ${sql5} """
+    assertTrue(explainStr5.toString().contains("percentile(profit, 0.5)"))
+    qt_select_5 "${sql5}"
+    qt_select_6 "${sql6}"
+
+    sql "DROP TABLE if exists sales"
+    sql "DROP DATABASE if exists test_convert_median_to_percentile"
+}

From f5279101862d7116a4ffad5349604b0de7dd23ed Mon Sep 17 00:00:00 2001
From: amory <wangqiannan@selectdb.com>
Date: Thu, 9 Jan 2025 17:25:56 +0800
Subject: [PATCH 081/140] [improve](ut)add data_type_array ut test and serde
 test (#46063)

### What problem does this PR solve?
this pr :
1/ data_type_array ut cover
2/ data_type_array_serde ut cover
---
 be/src/util/arrow/block_convertor.cpp         |   2 +-
 be/src/vec/data_types/data_type_array.cpp     |   4 +-
 .../serde/data_type_decimal_serde.cpp         |   5 +-
 .../serde/data_type_decimal_serde.h           |   4 +-
 .../serde/data_type_number_serde.cpp          |   5 +-
 .../data_types/common_data_type_serder_test.h |  61 +-
 .../vec/data_types/common_data_type_test.h    | 133 +++--
 .../vec/data_types/data_type_array_test.cpp   | 560 ++++++++++++++++++
 be/test/vec/data_types/data_type_ip_test.cpp  |  32 +-
 be/test/vec/function/function_test_util.cpp   |  57 ++
 10 files changed, 798 insertions(+), 65 deletions(-)
 create mode 100644 be/test/vec/data_types/data_type_array_test.cpp

diff --git a/be/src/util/arrow/block_convertor.cpp b/be/src/util/arrow/block_convertor.cpp
index eb2508c8d0cb74a..fadcef235f88c8c 100644
--- a/be/src/util/arrow/block_convertor.cpp
+++ b/be/src/util/arrow/block_convertor.cpp
@@ -392,7 +392,7 @@ Status FromBlockConverter::convert(std::shared_ptr<arrow::RecordBatch>* out) {
                                                           _timezone_obj);
         } catch (std::exception& e) {
             return Status::InternalError(
-                    "Fail to convert block data to arrow data, tyep: {}, name: {}, error: {}",
+                    "Fail to convert block data to arrow data, type: {}, name: {}, error: {}",
                     _cur_type->get_name(), e.what());
         }
         arrow_st = _cur_builder->Finish(&_arrays[_cur_field_idx]);
diff --git a/be/src/vec/data_types/data_type_array.cpp b/be/src/vec/data_types/data_type_array.cpp
index 254e13a9a8eea23..315ff8aee7431c5 100644
--- a/be/src/vec/data_types/data_type_array.cpp
+++ b/be/src/vec/data_types/data_type_array.cpp
@@ -65,8 +65,10 @@ bool DataTypeArray::equals(const IDataType& rhs) const {
            nested->equals(*static_cast<const DataTypeArray&>(rhs).nested);
 }
 
+// here we should remove nullable, otherwise here always be 1
 size_t DataTypeArray::get_number_of_dimensions() const {
-    const DataTypeArray* nested_array = typeid_cast<const DataTypeArray*>(nested.get());
+    const DataTypeArray* nested_array =
+            typeid_cast<const DataTypeArray*>(remove_nullable(nested).get());
     if (!nested_array) return 1;
     return 1 +
            nested_array
diff --git a/be/src/vec/data_types/serde/data_type_decimal_serde.cpp b/be/src/vec/data_types/serde/data_type_decimal_serde.cpp
index 32b08c3990fe16f..c4c33c36d5aad30 100644
--- a/be/src/vec/data_types/serde/data_type_decimal_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_decimal_serde.cpp
@@ -23,6 +23,7 @@
 #include <arrow/util/decimal.h>
 
 #include "arrow/type.h"
+#include "common/consts.h"
 #include "vec/columns/column_decimal.h"
 #include "vec/common/arithmetic_overflow.h"
 #include "vec/core/types.h"
@@ -109,7 +110,6 @@ void DataTypeDecimalSerDe<T>::write_column_to_arrow(const IColumn& column, const
             checkArrowStatus(builder.Append(value), column.get_name(),
                              array_builder->type()->name());
         }
-        // TODO: decimal256
     } else if constexpr (std::is_same_v<T, Decimal128V3>) {
         std::shared_ptr<arrow::DataType> s_decimal_ptr =
                 std::make_shared<arrow::Decimal128Type>(38, col.get_scale());
@@ -202,7 +202,8 @@ void DataTypeDecimalSerDe<T>::read_column_from_arrow(IColumn& column,
             column_data.emplace_back(*reinterpret_cast<const T*>(concrete_array->Value(value_i)));
         }
     } else {
-        LOG(WARNING) << "Unsuppoted convertion to decimal from " << column.get_name();
+        throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
+                               "read_column_from_arrow with type " + column.get_name());
     }
 }
 
diff --git a/be/src/vec/data_types/serde/data_type_decimal_serde.h b/be/src/vec/data_types/serde/data_type_decimal_serde.h
index 51867ced18fb04b..b29030051db5131 100644
--- a/be/src/vec/data_types/serde/data_type_decimal_serde.h
+++ b/be/src/vec/data_types/serde/data_type_decimal_serde.h
@@ -161,11 +161,11 @@ Status DataTypeDecimalSerDe<T>::write_column_to_pb(const IColumn& column, PValue
     return Status::OK();
 }
 
-// TODO: decimal256
 template <typename T>
 Status DataTypeDecimalSerDe<T>::read_column_from_pb(IColumn& column, const PValues& arg) const {
     if constexpr (std::is_same_v<T, Decimal<Int128>> || std::is_same_v<T, Decimal128V3> ||
-                  std::is_same_v<T, Decimal256> || std::is_same_v<T, Decimal<Int32>>) {
+                  std::is_same_v<T, Decimal256> || std::is_same_v<T, Decimal<Int32>> ||
+                  std::is_same_v<T, Decimal<Int64>>) {
         auto old_column_size = column.size();
         column.resize(old_column_size + arg.bytes_value_size());
         auto& data = reinterpret_cast<ColumnDecimal<T>&>(column).get_data();
diff --git a/be/src/vec/data_types/serde/data_type_number_serde.cpp b/be/src/vec/data_types/serde/data_type_number_serde.cpp
index 55c7b2c9505dae6..1ad919a6a94136e 100644
--- a/be/src/vec/data_types/serde/data_type_number_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_number_serde.cpp
@@ -210,7 +210,7 @@ void DataTypeNumberSerDe<T>::read_column_from_arrow(IColumn& column,
 
     // only for largeint(int128) type
     if (arrow_array->type_id() == arrow::Type::STRING) {
-        auto concrete_array = dynamic_cast<const arrow::StringArray*>(arrow_array);
+        const auto* concrete_array = dynamic_cast<const arrow::StringArray*>(arrow_array);
         std::shared_ptr<arrow::Buffer> buffer = concrete_array->value_data();
 
         for (size_t offset_i = start; offset_i < end; ++offset_i) {
@@ -226,6 +226,9 @@ void DataTypeNumberSerDe<T>::read_column_from_arrow(IColumn& column,
                                            std::string(rb.position(), rb.count()).c_str());
                 }
                 col_data.emplace_back(val);
+            } else {
+                // insert default value
+                col_data.emplace_back(Int128());
             }
         }
         return;
diff --git a/be/test/vec/data_types/common_data_type_serder_test.h b/be/test/vec/data_types/common_data_type_serder_test.h
index 46206d5ed7eb15d..4ecd5e1ca52b1af 100644
--- a/be/test/vec/data_types/common_data_type_serder_test.h
+++ b/be/test/vec/data_types/common_data_type_serder_test.h
@@ -24,22 +24,14 @@
 #include <fstream>
 #include <iostream>
 
-#include "olap/schema.h"
-#include "runtime/descriptors.cpp"
+#include "arrow/type.h"
 #include "runtime/descriptors.h"
 #include "util/arrow/block_convertor.h"
 #include "util/arrow/row_batch.h"
 #include "vec/columns/column.h"
-#include "vec/columns/column_array.h"
-#include "vec/columns/column_map.h"
-#include "vec/columns/columns_number.h"
 #include "vec/core/field.h"
-#include "vec/core/sort_block.h"
-#include "vec/core/sort_description.h"
 #include "vec/core/types.h"
 #include "vec/data_types/data_type.h"
-#include "vec/data_types/data_type_array.h"
-#include "vec/data_types/data_type_map.h"
 #include "vec/utils/arrow_column_to_doris_column.h"
 
 // this test is gonna to be a data type serialize and deserialize functions
@@ -151,16 +143,14 @@ class CommonDataTypeSerdeTest : public ::testing::Test {
 
         while (std::getline(file, line)) {
             std::stringstream lineStream(line);
-            //            std::cout << "whole : " << lineStream.str() << std::endl;
             std::string value;
             int l_idx = 0;
             int c_idx = 0;
-            std::vector<string> row;
+            std::vector<std::string> row;
             while (std::getline(lineStream, value, spliter)) {
-                if (idxes.contains(l_idx)) {
+                if (!value.starts_with("//") && idxes.contains(l_idx)) {
                     // load csv data
                     Slice string_slice(value.data(), value.size());
-                    std::cout << "origin : " << string_slice << std::endl;
                     Status st;
                     // deserialize data
                     if constexpr (is_hive_format) {
@@ -210,7 +200,7 @@ class CommonDataTypeSerdeTest : public ::testing::Test {
         if (generate_res_file) {
             // generate res
             auto pos = file_path.find_last_of(".");
-            string hive_format = is_hive_format ? "_hive" : "";
+            std::string hive_format = is_hive_format ? "_hive" : "";
             std::string res_file = file_path.substr(0, pos) + hive_format + "_serde_res.csv";
             std::ofstream res_f(res_file);
             if (!res_f.is_open()) {
@@ -218,8 +208,6 @@ class CommonDataTypeSerdeTest : public ::testing::Test {
             }
             for (size_t r = 0; r < assert_str_cols[0]->size(); ++r) {
                 for (size_t c = 0; c < assert_str_cols.size(); ++c) {
-                    std::cout << assert_str_cols[c]->get_data_at(r).to_string() << spliter
-                              << std::endl;
                     res_f << assert_str_cols[c]->get_data_at(r).to_string() << spliter;
                 }
                 res_f << std::endl;
@@ -230,6 +218,8 @@ class CommonDataTypeSerdeTest : public ::testing::Test {
     }
 
     // standard hive text ser-deserialize assert function
+    // pb serde now is only used RPCFncall and fold_constant_executor which just write column data to pb value means
+    // just call write_column_to_pb
     static void assert_pb_format(MutableColumns& load_cols, DataTypeSerDeSPtrs serders) {
         for (size_t i = 0; i < load_cols.size(); ++i) {
             auto& col = load_cols[i];
@@ -263,6 +253,21 @@ class CommonDataTypeSerdeTest : public ::testing::Test {
     static void assert_jsonb_format(MutableColumns& load_cols, DataTypeSerDeSPtrs serders) {
         Arena pool;
         auto jsonb_column = ColumnString::create(); // jsonb column
+        // maybe these load_cols has different size, so we keep it same
+        size_t max_row_size = load_cols[0]->size();
+        for (size_t i = 1; i < load_cols.size(); ++i) {
+            if (load_cols[i]->size() > max_row_size) {
+                max_row_size = load_cols[i]->size();
+            }
+        }
+        // keep same rows
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            if (load_cols[i]->size() < max_row_size) {
+                load_cols[i]->insert_many_defaults(max_row_size - load_cols[i]->size());
+            } else if (load_cols[i]->size() > max_row_size) {
+                load_cols[i]->resize(max_row_size);
+            }
+        }
         jsonb_column->reserve(load_cols[0]->size());
         MutableColumns assert_cols;
         for (size_t i = 0; i < load_cols.size(); ++i) {
@@ -322,6 +327,21 @@ class CommonDataTypeSerdeTest : public ::testing::Test {
                                     DataTypes types) {
         // make a block to write to arrow
         auto block = std::make_shared<Block>();
+        // maybe these load_cols has different size, so we keep it same
+        size_t max_row_size = load_cols[0]->size();
+        for (size_t i = 1; i < load_cols.size(); ++i) {
+            if (load_cols[i]->size() > max_row_size) {
+                max_row_size = load_cols[i]->size();
+            }
+        }
+        // keep same rows
+        for (size_t i = 0; i < load_cols.size(); ++i) {
+            if (load_cols[i]->size() < max_row_size) {
+                load_cols[i]->insert_many_defaults(max_row_size - load_cols[i]->size());
+            } else if (load_cols[i]->size() > max_row_size) {
+                load_cols[i]->resize(max_row_size);
+            }
+        }
         for (size_t i = 0; i < load_cols.size(); ++i) {
             auto& col = load_cols[i];
             block->insert(ColumnWithTypeAndName(std::move(col), types[i], types[i]->get_name()));
@@ -330,13 +350,13 @@ class CommonDataTypeSerdeTest : public ::testing::Test {
         std::cout << "block: " << block->dump_structure() << std::endl;
         std::shared_ptr<arrow::Schema> block_arrow_schema;
         EXPECT_EQ(get_arrow_schema_from_block(*block, &block_arrow_schema, "UTC"), Status::OK());
+        std::cout << "schema: " << block_arrow_schema->ToString(true) << std::endl;
         // convert block to arrow
         std::shared_ptr<arrow::RecordBatch> result;
         cctz::time_zone _timezone_obj; //default UTC
         Status stt = convert_to_arrow_batch(*block, block_arrow_schema,
                                             arrow::default_memory_pool(), &result, _timezone_obj);
         EXPECT_EQ(Status::OK(), stt) << "convert block to arrow failed" << stt.to_string();
-
         // deserialize arrow to block
         auto assert_block = block->clone_empty();
         auto rows = block->rows();
@@ -347,15 +367,22 @@ class CommonDataTypeSerdeTest : public ::testing::Test {
                     array.get(), 0, column_with_type_and_name.column,
                     column_with_type_and_name.type, rows, _timezone_obj);
             // do check data
+            std::cout << "arrow_column_to_doris_column done: "
+                      << column_with_type_and_name.column->get_name()
+                      << " with column size: " << column_with_type_and_name.column->size()
+                      << std::endl;
+            std::cout << assert_block.dump_structure() << std::endl;
             EXPECT_EQ(Status::OK(), ret) << "convert arrow to block failed" << ret.to_string();
             auto& col = block->get_by_position(i).column;
             auto& assert_col = column_with_type_and_name.column;
+            EXPECT_EQ(assert_col->size(), col->size());
             for (size_t j = 0; j < col->size(); ++j) {
                 auto cell = col->operator[](j);
                 auto assert_cell = assert_col->operator[](j);
                 EXPECT_EQ(cell, assert_cell) << "column: " << col->get_name() << " row: " << j;
             }
         }
+        std::cout << "assert block: " << assert_block.dump_structure() << std::endl;
     }
 
     // assert rapidjson format
diff --git a/be/test/vec/data_types/common_data_type_test.h b/be/test/vec/data_types/common_data_type_test.h
index 36abc3402e21561..d2e42ce50cdbf56 100644
--- a/be/test/vec/data_types/common_data_type_test.h
+++ b/be/test/vec/data_types/common_data_type_test.h
@@ -82,14 +82,13 @@ class CommonDataTypeTest : public ::testing::Test {
             int l_idx = 0;
             int c_idx = 0;
             while (std::getline(lineStream, value, spliter)) {
-                if (idxes.contains(l_idx)) {
+                if ((!value.starts_with("//") && idxes.contains(l_idx))) {
                     Slice string_slice(value.data(), value.size());
-                    std::cout << string_slice << std::endl;
                     if (auto st = serders[c_idx]->deserialize_one_cell_from_json(
                                 *columns[c_idx], string_slice, options);
                         !st.ok()) {
-                        //                        std::cout << "error in deserialize but continue: " << st.to_string()
-                        //                                  << std::endl;
+                        std::cout << "error in deserialize but continue: " << st.to_string()
+                                  << std::endl;
                     }
                     ++c_idx;
                 }
@@ -116,7 +115,8 @@ class CommonDataTypeTest : public ::testing::Test {
         bool is_value_represented_by_number = false;
         PColumnMeta* pColumnMeta = nullptr;
         DataTypeSerDeSPtr serde = nullptr;
-        //        bool is_value_unambiguously_represented_in_contiguous_memory_region = false;
+        bool is_value_unambiguously_represented_in_contiguous_memory_region = false;
+        Field default_field;
     };
     void SetUp() override {}
 
@@ -134,7 +134,15 @@ class CommonDataTypeTest : public ::testing::Test {
         ASSERT_EQ(data_type->text_can_contain_only_valid_utf8(),
                   meta_info.text_can_contain_only_valid_utf8);
         ASSERT_EQ(data_type->have_maximum_size_of_value(), meta_info.have_maximum_size_of_value);
-        ASSERT_EQ(data_type->get_size_of_value_in_memory(), meta_info.size_of_value_in_memory);
+        ASSERT_EQ(data_type->is_value_unambiguously_represented_in_contiguous_memory_region(),
+                  meta_info.is_value_unambiguously_represented_in_contiguous_memory_region);
+        if (is_decimal(data_type) || data_type->is_nullable() || is_struct(data_type) ||
+            is_nothing(data_type) || is_number(data_type) || is_columned_as_number(data_type) ||
+            is_ip(data_type)) {
+            ASSERT_EQ(data_type->get_size_of_value_in_memory(), meta_info.size_of_value_in_memory);
+        } else {
+            EXPECT_ANY_THROW(EXPECT_FALSE(data_type->get_size_of_value_in_memory()));
+        }
         if (is_decimal(data_type)) {
             ASSERT_EQ(data_type->get_precision(), meta_info.precision);
             ASSERT_EQ(data_type->get_scale(), meta_info.scale);
@@ -145,11 +153,13 @@ class CommonDataTypeTest : public ::testing::Test {
         ASSERT_EQ(data_type->is_null_literal(), meta_info.is_null_literal);
         ASSERT_EQ(data_type->is_value_represented_by_number(),
                   meta_info.is_value_represented_by_number);
-        //        ASSERT_EQ(data_type->is_value_unambiguously_represented_in_contiguous_memory_region(), meta_info.is_value_unambiguously_represented_in_contiguous_memory_region);
+        ASSERT_EQ(data_type->get_default(), meta_info.default_field);
     }
 
     // create column assert with default field is simple and can be used for all DataType
-    void create_column_assert(DataTypePtr& data_type, Field& default_field) {
+    void create_column_assert(DataTypePtr& data_type, Field& default_field,
+                              size_t uncompressed_serialized_bytes = -1) {
+        std::cout << "create_column_assert: " << data_type->get_name() << std::endl;
         auto column = data_type->create_column();
         ASSERT_EQ(column->size(), 0);
         ColumnPtr const_col = data_type->create_column_const(10, default_field);
@@ -160,7 +170,8 @@ class CommonDataTypeTest : public ::testing::Test {
             ASSERT_EQ(const_col->operator[](i), default_const_col->operator[](i));
         }
         // get_uncompressed_serialized_bytes
-        ASSERT_EQ(data_type->get_uncompressed_serialized_bytes(*column, 0), 4);
+        ASSERT_EQ(data_type->get_uncompressed_serialized_bytes(*column, 0),
+                  uncompressed_serialized_bytes);
     }
 
     // get_field assert is simple and can be used for all DataType
@@ -183,33 +194,87 @@ class CommonDataTypeTest : public ::testing::Test {
     // to_string | to_string_batch | from_string assert is simple and can be used for all DataType
     void assert_to_string_from_string_assert(MutableColumnPtr mutableColumn,
                                              DataTypePtr& data_type) {
-        // to_string_batch | from_string
-        auto col_to = ColumnString::create();
-        data_type->to_string_batch(*mutableColumn, *col_to);
-        ASSERT_EQ(col_to->size(), mutableColumn->size());
-        // from_string assert col_to to assert_column and check same with mutableColumn
-        auto assert_column = data_type->create_column();
-        for (int i = 0; i < col_to->size(); ++i) {
-            std::string s = col_to->get_data_at(i).to_string();
-            ReadBuffer rb(s.data(), s.size());
-            ASSERT_EQ(Status::OK(), data_type->from_string(rb, assert_column.get()));
-            ASSERT_EQ(assert_column->operator[](i), mutableColumn->operator[](i));
+        {
+            // to_string_batch | from_string
+            auto col_to = ColumnString::create();
+            data_type->to_string_batch(*mutableColumn, *col_to);
+            ASSERT_EQ(col_to->size(), mutableColumn->size());
+            // from_string assert col_to to assert_column and check same with mutableColumn
+            auto assert_column = data_type->create_column();
+            for (int i = 0; i < col_to->size(); ++i) {
+                std::string s = col_to->get_data_at(i).to_string();
+                ReadBuffer rb(s.data(), s.size());
+                ASSERT_EQ(Status::OK(), data_type->from_string(rb, assert_column.get()));
+                ASSERT_EQ(assert_column->operator[](i), mutableColumn->operator[](i))
+                        << "i: " << i << " s: " << s << " datatype: " << data_type->get_name()
+                        << " assert_column: " << assert_column->get_name()
+                        << " mutableColumn:" << mutableColumn->get_name() << std::endl;
+            }
+        }
+        {
+            std::cout << "assert to_string from_string is reciprocal: " << data_type->get_name()
+                      << std::endl;
+            // to_string | from_string
+            auto ser_col = ColumnString::create();
+            ser_col->reserve(mutableColumn->size());
+            VectorBufferWriter buffer_writer(*ser_col.get());
+            for (int i = 0; i < mutableColumn->size(); ++i) {
+                data_type->to_string(*mutableColumn, i, buffer_writer);
+                std::string res = data_type->to_string(*mutableColumn, i);
+                buffer_writer.commit();
+                EXPECT_EQ(res, ser_col->get_data_at(i).to_string());
+            }
+            // check ser_col to assert_column and check same with mutableColumn
+            auto assert_column_1 = data_type->create_column();
+            for (int i = 0; i < ser_col->size(); ++i) {
+                std::string s = ser_col->get_data_at(i).to_string();
+                ReadBuffer rb(s.data(), s.size());
+                ASSERT_EQ(Status::OK(), data_type->from_string(rb, assert_column_1.get()));
+                ASSERT_EQ(assert_column_1->operator[](i), mutableColumn->operator[](i));
+            }
         }
-        // to_string | from_string
-        auto ser_col = ColumnString::create();
-        ser_col->reserve(mutableColumn->size());
-        VectorBufferWriter buffer_writer(*ser_col.get());
-        for (int i = 0; i < mutableColumn->size(); ++i) {
-            data_type->to_string(*mutableColumn, i, buffer_writer);
-            buffer_writer.commit();
+    }
+
+    // datatype serialize | deserialize assert is only used Block::serialize | deserialize which for PBlock
+    //  which happened in multiple BE shuffle data
+    void serialize_deserialize_assert(MutableColumns& columns, DataTypes data_types) {
+        // first make columns has same rows
+        size_t max_row = columns[0]->size();
+        for (int i = 1; i < columns.size(); ++i) {
+            max_row = std::max(max_row, columns[i]->size());
+        }
+        for (int i = 0; i < columns.size(); ++i) {
+            if (columns[i]->size() < max_row) {
+                columns[i]->resize(max_row);
+            }
+        }
+        // wrap columns into block
+        auto block = std::make_shared<Block>();
+        for (int i = 0; i < columns.size(); ++i) {
+            block->insert({columns[i]->get_ptr(), data_types[i], ""});
         }
-        // check ser_col to assert_column and check same with mutableColumn
-        auto assert_column_1 = data_type->create_column();
-        for (int i = 0; i < ser_col->size(); ++i) {
-            std::string s = ser_col->get_data_at(i).to_string();
-            ReadBuffer rb(s.data(), s.size());
-            ASSERT_EQ(Status::OK(), data_type->from_string(rb, assert_column_1.get()));
-            ASSERT_EQ(assert_column_1->operator[](i), mutableColumn->operator[](i));
+        // nt be_exec_version, PBlock* pblock, size_t* uncompressed_bytes,
+        //                     size_t* compressed_bytes, segment_v2::CompressionTypePB compression_type,
+        size_t be_exec_version = 2;
+        auto pblock = std::make_unique<PBlock>();
+        size_t uncompressed_bytes = 0;
+        size_t compressed_bytes = 0;
+        segment_v2::CompressionTypePB compression_type = segment_v2::CompressionTypePB::ZSTD;
+        Status st = block->serialize(be_exec_version, pblock.get(), &uncompressed_bytes,
+                                     &compressed_bytes, compression_type);
+        ASSERT_EQ(st.ok(), true);
+        // deserialize
+        auto block_1 = std::make_shared<Block>();
+        st = block_1->deserialize(*pblock);
+        ASSERT_EQ(st.ok(), true);
+        // check block_1 and block is same
+        for (int i = 0; i < block->rows(); ++i) {
+            auto& col = block->get_by_position(i);
+            auto& col_1 = block_1->get_by_position(i);
+            ASSERT_EQ(col.column->size(), col_1.column->size());
+            for (int j = 0; j < col.column->size(); ++j) {
+                ASSERT_EQ(col.column->operator[](j), col_1.column->operator[](j));
+            }
         }
     }
 
diff --git a/be/test/vec/data_types/data_type_array_test.cpp b/be/test/vec/data_types/data_type_array_test.cpp
new file mode 100644
index 000000000000000..6e643bcdf23a6f6
--- /dev/null
+++ b/be/test/vec/data_types/data_type_array_test.cpp
@@ -0,0 +1,560 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <execinfo.h> // for backtrace on Linux
+#include <gtest/gtest-message.h>
+#include <gtest/gtest-test-part.h>
+#include <gtest/gtest.h>
+
+#include <filesystem>
+#include <iostream>
+#include <stdexcept>
+
+#include "vec/columns/column.h"
+#include "vec/columns/columns_number.h"
+#include "vec/core/field.h"
+#include "vec/core/types.h"
+#include "vec/data_types/common_data_type_serder_test.h"
+#include "vec/data_types/common_data_type_test.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/function/function_test_util.h"
+
+// this test is gonna to be a data type test template for all DataType which should make ut test to coverage the function defined
+// for example DataTypeIPv4 should test this function:
+// 1. datatype meta info:
+//         get_type_id, get_type_as_type_descriptor, get_storage_field_type, have_subtypes, get_pdata_type (const IDataType *data_type), to_pb_column_meta (PColumnMeta *col_meta)
+//         get_family_name, get_is_parametric, should_align_right_in_pretty_formats
+//         text_can_contain_only_valid_utf8
+//         have_maximum_size_of_value, get_maximum_size_of_value_in_memory, get_size_of_value_in_memory
+//         get_precision, get_scale
+//         is_null_literal, is_value_represented_by_number, is_value_unambiguously_represented_in_contiguous_memory_region
+// 2. datatype creation with column : create_column, create_column_const (size_t size, const Field &field), create_column_const_with_default_value (size_t size), get_uncompressed_serialized_bytes (const IColumn &column, int be_exec_version)
+// 3. serde related: get_serde (int nesting_level=1)
+//          to_string (const IColumn &column, size_t row_num, BufferWritable &ostr), to_string (const IColumn &column, size_t row_num), to_string_batch (const IColumn &column, ColumnString &column_to), from_string (ReadBuffer &rb, IColumn *column)
+//          serialize (const IColumn &column, char *buf, int be_exec_version), deserialize (const char *buf, MutableColumnPtr *column, int be_exec_version)
+// 4. compare: equals (const IDataType &rhs), is_comparable
+// 5. others: update_avg_value_size_hint (const IColumn &column, double &avg_value_size_hint)
+
+namespace doris::vectorized {
+
+class DataTypeArrayTest : public CommonDataTypeTest {
+protected:
+    void SetUp() override {
+        // insert from data csv and assert insert result
+        MutableColumns array_cols;
+        // we need to load data from csv file into column_array list
+        // step1. create data type for array nested type (const and nullable)
+        // array<tinyint>
+        BaseInputTypeSet array_tinyint = {TypeIndex::Array, TypeIndex::Int8};
+        // array<smallint>
+        BaseInputTypeSet array_smallint = {TypeIndex::Array, TypeIndex::Int16};
+        // array<int>
+        BaseInputTypeSet array_int = {TypeIndex::Array, TypeIndex::Int32};
+        // array<bigint>
+        BaseInputTypeSet array_bigint = {TypeIndex::Array, TypeIndex::Int64};
+        // array<largeint>
+        BaseInputTypeSet array_largeint = {TypeIndex::Array, TypeIndex::Int128};
+        // array<float>
+        BaseInputTypeSet array_float = {TypeIndex::Array, TypeIndex::Float32};
+        // array<double>
+        BaseInputTypeSet array_double = {TypeIndex::Array, TypeIndex::Float64};
+        // array<ipv4>
+        BaseInputTypeSet array_ipv4 = {TypeIndex::Array, TypeIndex::IPv4};
+        // array<ipv6>
+        BaseInputTypeSet array_ipv6 = {TypeIndex::Array, TypeIndex::IPv6};
+        // array<date>
+        BaseInputTypeSet array_date = {TypeIndex::Array, TypeIndex::Date};
+        // array<datetime>
+        BaseInputTypeSet array_datetime = {TypeIndex::Array, TypeIndex::DateTime};
+        // array<datev2>
+        BaseInputTypeSet array_datev2 = {TypeIndex::Array, TypeIndex::DateV2};
+        // array<datetimev2>
+        BaseInputTypeSet array_datetimev2 = {TypeIndex::Array, TypeIndex::DateTimeV2};
+        // array<varchar>
+        BaseInputTypeSet array_varchar = {TypeIndex::Array, TypeIndex::String};
+        // array<decimal32(9, 5)> UT
+        BaseInputTypeSet array_decimal = {TypeIndex::Array, TypeIndex::Decimal32};
+        // array<decimal64(18, 9)> UT
+        BaseInputTypeSet array_decimal64 = {TypeIndex::Array, TypeIndex::Decimal64};
+        // array<decimal128(38, 20)> UT
+        BaseInputTypeSet array_decimal128 = {TypeIndex::Array, TypeIndex::Decimal128V3};
+        // array<decimal256(76, 40)> UT
+        BaseInputTypeSet array_decimal256 = {TypeIndex::Array, TypeIndex::Decimal256};
+        std::vector<BaseInputTypeSet> array_typeIndex = {
+                array_tinyint,   array_smallint,   array_int,        array_bigint,  array_largeint,
+                array_float,     array_double,     array_ipv4,       array_ipv6,    array_date,
+                array_datetime,  array_datev2,     array_datetimev2, array_varchar, array_decimal,
+                array_decimal64, array_decimal128, array_decimal256};
+        // array<array<tinyint>>
+        BaseInputTypeSet array_array_tinyint = {TypeIndex::Array, TypeIndex::Array,
+                                                TypeIndex::Int8};
+        // array<array<smallint>>
+        BaseInputTypeSet array_array_smallint = {TypeIndex::Array, TypeIndex::Array,
+                                                 TypeIndex::Int16};
+        // array<array<int>>
+        BaseInputTypeSet array_array_int = {TypeIndex::Array, TypeIndex::Array, TypeIndex::Int32};
+        // array<array<bigint>>
+        BaseInputTypeSet array_array_bigint = {TypeIndex::Array, TypeIndex::Array,
+                                               TypeIndex::Int64};
+        // array<array<largeint>>
+        BaseInputTypeSet array_array_largeint = {TypeIndex::Array, TypeIndex::Array,
+                                                 TypeIndex::Int128};
+        // array<array<float>>
+        BaseInputTypeSet array_array_float = {TypeIndex::Array, TypeIndex::Array,
+                                              TypeIndex::Float32};
+        // array<array<double>>
+        BaseInputTypeSet array_array_double = {TypeIndex::Array, TypeIndex::Array,
+                                               TypeIndex::Float64};
+        // array<array<ipv4>>
+        BaseInputTypeSet array_array_ipv4 = {TypeIndex::Array, TypeIndex::Array, TypeIndex::IPv4};
+        // array<array<ipv6>>
+        BaseInputTypeSet array_array_ipv6 = {TypeIndex::Array, TypeIndex::Array, TypeIndex::IPv6};
+        // array<array<date>>
+        BaseInputTypeSet array_array_date = {TypeIndex::Array, TypeIndex::Array, TypeIndex::Date};
+        // array<array<datetime>>
+        BaseInputTypeSet array_array_datetime = {TypeIndex::Array, TypeIndex::Array,
+                                                 TypeIndex::DateTime};
+        // array<array<datev2>>
+        BaseInputTypeSet array_array_datev2 = {TypeIndex::Array, TypeIndex::Array,
+                                               TypeIndex::DateV2};
+        // array<array<datetimev2>>
+        BaseInputTypeSet array_array_datetimev2 = {TypeIndex::Array, TypeIndex::Array,
+                                                   TypeIndex::DateTimeV2};
+        // array<array<varchar>>
+        BaseInputTypeSet array_array_varchar = {TypeIndex::Array, TypeIndex::Array,
+                                                TypeIndex::String};
+        // array<array<decimal32(9, 5)>> UT
+        BaseInputTypeSet array_array_decimal = {TypeIndex::Array, TypeIndex::Array,
+                                                TypeIndex::Decimal32};
+        // array<array<decimal64(18, 9)>> UT
+        BaseInputTypeSet array_array_decimal64 = {TypeIndex::Array, TypeIndex::Array,
+                                                  TypeIndex::Decimal64};
+        // array<array<decimal128(38, 20)>> UT
+        BaseInputTypeSet array_array_decimal128 = {TypeIndex::Array, TypeIndex::Array,
+                                                   TypeIndex::Decimal128V3};
+        // array<array<decimal256(76, 40)>> UT
+        BaseInputTypeSet array_array_decimal256 = {TypeIndex::Array, TypeIndex::Array,
+                                                   TypeIndex::Decimal256};
+        // array<map<char,double>>
+        BaseInputTypeSet array_map_char_double = {TypeIndex::Array, TypeIndex::Map,
+                                                  TypeIndex::String, TypeIndex::Float64};
+        // test_array_map<datetime,decimal<76,56>>.csv
+        BaseInputTypeSet array_map_datetime_decimal = {TypeIndex::Array, TypeIndex::Map,
+                                                       TypeIndex::DateTime, TypeIndex::Decimal256};
+        // test_array_map<ipv4,ipv6>.csv
+        BaseInputTypeSet array_map_ipv4_ipv6 = {TypeIndex::Array, TypeIndex::Map, TypeIndex::IPv4,
+                                                TypeIndex::IPv6};
+        // test_array_map<largeInt,string>.csv
+        BaseInputTypeSet array_map_largeint_string = {TypeIndex::Array, TypeIndex::Map,
+                                                      TypeIndex::Int128, TypeIndex::String};
+        // array<struct<f1:int,f2:date,f3:decimal,f4:string,f5:double,f6:ipv4,f7:ipv6>>
+        BaseInputTypeSet array_struct = {
+                TypeIndex::Array,   TypeIndex::Struct,    TypeIndex::Int32,
+                TypeIndex::Date,    TypeIndex::Decimal32, TypeIndex::String,
+                TypeIndex::Float64, TypeIndex::IPv4,      TypeIndex::IPv6};
+
+        std::vector<BaseInputTypeSet> array_array_typeIndex = {
+                array_array_tinyint,    array_array_smallint,   array_array_int,
+                array_array_bigint,     array_array_largeint,   array_array_float,
+                array_array_double,     array_array_ipv4,       array_array_ipv6,
+                array_array_date,       array_array_datetime,   array_array_datev2,
+                array_array_datetimev2, array_array_varchar,    array_array_decimal,
+                array_array_decimal64,  array_array_decimal128, array_array_decimal256};
+        std::vector<BaseInputTypeSet> array_map_typeIndex = {
+                array_map_char_double, array_map_datetime_decimal, array_map_ipv4_ipv6,
+                array_map_largeint_string};
+        std::vector<BaseInputTypeSet> array_struct_typeIndex = {array_struct};
+
+        array_descs.reserve(array_typeIndex.size() + array_array_typeIndex.size() +
+                            array_map_typeIndex.size() + array_struct_typeIndex.size());
+        for (int i = 0; i < array_typeIndex.size(); i++) {
+            array_descs.push_back(ut_type::UTDataTypeDescs());
+            InputTypeSet input_types {};
+            input_types.push_back(array_typeIndex[i][0]);
+            input_types.push_back(Nullable {static_cast<TypeIndex>(array_typeIndex[i][1])});
+            EXPECT_EQ(input_types[1].type(), &typeid(Nullable)) << "nested type is not nullable";
+            EXPECT_TRUE(parse_ut_data_type(input_types, array_descs[i]));
+        }
+        for (int i = 0; i < array_array_typeIndex.size(); i++) {
+            array_descs.push_back(ut_type::UTDataTypeDescs());
+            InputTypeSet input_types {};
+            input_types.push_back(array_array_typeIndex[i][0]);
+            input_types.push_back(Nullable {static_cast<TypeIndex>(array_array_typeIndex[i][1])});
+            input_types.push_back(Nullable {static_cast<TypeIndex>(array_array_typeIndex[i][2])});
+            EXPECT_EQ(input_types[1].type(), &typeid(Nullable)) << "nested type is not nullable";
+            EXPECT_EQ(input_types[2].type(), &typeid(Nullable)) << "nested type is not nullable";
+            EXPECT_TRUE(parse_ut_data_type(input_types, array_descs[i + array_typeIndex.size()]));
+        }
+
+        for (int i = 0; i < array_map_typeIndex.size(); i++) {
+            array_descs.push_back(ut_type::UTDataTypeDescs());
+            InputTypeSet input_types {};
+            input_types.push_back(array_map_typeIndex[i][0]); // array
+            input_types.push_back(
+                    Nullable {static_cast<TypeIndex>(array_map_typeIndex[i][1])}); // map
+            input_types.push_back(
+                    Nullable {static_cast<TypeIndex>(array_map_typeIndex[i][2])}); // key
+            input_types.push_back(
+                    Nullable {static_cast<TypeIndex>(array_map_typeIndex[i][3])}); // val
+            EXPECT_EQ(input_types[1].type(), &typeid(Nullable)) << "nested type is not nullable";
+            EXPECT_EQ(input_types[2].type(), &typeid(Nullable)) << "nested type is not nullable";
+            EXPECT_TRUE(parse_ut_data_type(
+                    input_types,
+                    array_descs[i + array_typeIndex.size() + array_array_typeIndex.size()]));
+        }
+
+        for (int i = 0; i < array_struct_typeIndex.size(); i++) {
+            array_descs.push_back(ut_type::UTDataTypeDescs());
+            InputTypeSet input_types {};
+            input_types.push_back(array_struct_typeIndex[i][0]); // arr
+            input_types.push_back(
+                    Nullable {static_cast<TypeIndex>(array_struct_typeIndex[i][1])}); // struct
+            input_types.push_back(
+                    Nullable {static_cast<TypeIndex>(array_struct_typeIndex[i][2])}); // f1
+            input_types.push_back(
+                    Nullable {static_cast<TypeIndex>(array_struct_typeIndex[i][3])}); // f2
+            input_types.push_back(
+                    Nullable {static_cast<TypeIndex>(array_struct_typeIndex[i][4])}); // f3
+            input_types.push_back(
+                    Nullable {static_cast<TypeIndex>(array_struct_typeIndex[i][5])}); // f4
+            input_types.push_back(
+                    Nullable {static_cast<TypeIndex>(array_struct_typeIndex[i][6])}); // f5
+            input_types.push_back(
+                    Nullable {static_cast<TypeIndex>(array_struct_typeIndex[i][7])}); // f6
+            input_types.push_back(
+                    Nullable {static_cast<TypeIndex>(array_struct_typeIndex[i][8])}); // f7
+
+            EXPECT_EQ(input_types[1].type(), &typeid(Nullable)) << "nested type is not nullable";
+            EXPECT_TRUE(parse_ut_data_type(
+                    input_types,
+                    array_descs[i + array_typeIndex.size() + array_array_typeIndex.size() +
+                                array_map_typeIndex.size()]));
+        }
+
+        // create column_array for each data type
+        // step2. according to the datatype to make column_array
+        //          && load data from csv file into column_array
+        EXPECT_EQ(array_descs.size(), data_files.size());
+        for (int i = 0; i < array_descs.size(); i++) {
+            auto& desc = array_descs[i];
+            auto& data_file = data_files[i];
+            // first is array type
+            auto& type = desc[0].data_type;
+            std::cout << "type: " << type->get_name() << " with file: " << data_file << std::endl;
+            MutableColumns columns;
+            columns.push_back(type->create_column());
+            auto serde = type->get_serde(1);
+            load_data_from_csv({serde}, columns, data_file, ';');
+            array_columns.push_back(std::move(columns[0]));
+            array_types.push_back(type);
+            serdes.push_back(serde);
+        }
+    }
+
+    std::string data_file_dir = "regression-test/data/nereids_function_p0/array/";
+
+    vector<string> data_files = {
+            // array-scalar
+            data_file_dir + "test_array_tinyint.csv", data_file_dir + "test_array_smallint.csv",
+            data_file_dir + "test_array_int.csv", data_file_dir + "test_array_bigint.csv",
+            data_file_dir + "test_array_largeint.csv", data_file_dir + "test_array_float.csv",
+            data_file_dir + "test_array_double.csv", data_file_dir + "test_array_ipv4.csv",
+            data_file_dir + "test_array_ipv6.csv", data_file_dir + "test_array_date.csv",
+            data_file_dir + "test_array_datetime.csv", data_file_dir + "test_array_date.csv",
+            data_file_dir + "test_array_datetimev2(6).csv",
+            data_file_dir + "test_array_varchar(65535).csv",
+            data_file_dir + "test_array_decimalv3(7,4).csv",
+            data_file_dir + "test_array_decimalv3(16,10).csv",
+            data_file_dir + "test_array_decimalv3(38,30).csv",
+            data_file_dir + "test_array_decimalv3(76,56).csv",
+            // array-array
+            data_file_dir + "test_array_array_tinyint.csv",
+            data_file_dir + "test_array_array_smallint.csv",
+            data_file_dir + "test_array_array_int.csv",
+            data_file_dir + "test_array_array_bigint.csv",
+            data_file_dir + "test_array_array_largeint.csv",
+            data_file_dir + "test_array_array_float.csv",
+            data_file_dir + "test_array_array_double.csv",
+            data_file_dir + "test_array_array_ipv4.csv",
+            data_file_dir + "test_array_array_ipv6.csv",
+            data_file_dir + "test_array_array_date.csv",
+            data_file_dir + "test_array_array_datetime.csv",
+            data_file_dir + "test_array_array_date.csv",
+            data_file_dir + "test_array_array_datetimev2(5).csv",
+            data_file_dir + "test_array_array_varchar(65535).csv",
+            data_file_dir + "test_array_array_decimalv3(1,0).csv",
+            data_file_dir + "test_array_array_decimalv3(27,9).csv",
+            data_file_dir + "test_array_array_decimalv3(38,30).csv",
+            data_file_dir + "test_array_array_decimalv3(76,56).csv",
+            // array-map - 36
+            data_file_dir + "test_array_map<char,double>.csv",
+            data_file_dir + "test_array_map<datetime,decimal<76,56>>.csv",
+            data_file_dir + "test_array_map<ipv4,ipv6>.csv",
+            data_file_dir + "test_array_map<largeInt,string>.csv",
+            // array-struct
+            data_file_dir + "test_array_struct.csv"};
+
+    vector<ut_type::UTDataTypeDescs> array_descs; // array<> descs matrix
+    MutableColumns array_columns;                 // column_array list
+    DataTypes array_types;
+    DataTypeSerDeSPtrs serdes;
+};
+
+// ================ common data type and serder test =================
+TEST_F(DataTypeArrayTest, MetaInfoTest) {
+    for (int i = 0; i < array_types.size(); i++) {
+        auto& type = array_types[i];
+        auto& desc = array_descs[i];
+        auto array_type = assert_cast<const DataTypeArray*>(remove_nullable(type).get());
+        auto nested_type =
+                assert_cast<const DataTypeArray*>(remove_nullable(type).get())->get_nested_type();
+
+        TypeDescriptor arr_type_descriptor = {PrimitiveType::TYPE_ARRAY};
+        arr_type_descriptor.add_sub_type(desc[0].type_desc.children[0]);
+        auto col_meta = std::make_shared<PColumnMeta>();
+        array_type->to_pb_column_meta(col_meta.get());
+        Array a;
+        a.push_back(nested_type->get_default());
+
+        DataTypeMetaInfo arr_meta_info_to_assert = {
+                .type_id = TypeIndex::Array,
+                .type_as_type_descriptor = &arr_type_descriptor,
+                .family_name = "Array",
+                .has_subtypes = true,
+                .storage_field_type = doris::FieldType::OLAP_FIELD_TYPE_ARRAY,
+                .should_align_right_in_pretty_formats = false,
+                .text_can_contain_only_valid_utf8 = nested_type->text_can_contain_only_valid_utf8(),
+                .have_maximum_size_of_value = false,
+                .size_of_value_in_memory = size_t(-1),
+                .precision = size_t(-1),
+                .scale = size_t(-1),
+                .is_null_literal = false,
+                .is_value_represented_by_number = false,
+                .pColumnMeta = col_meta.get(),
+                .is_value_unambiguously_represented_in_contiguous_memory_region =
+                        nested_type
+                                ->is_value_unambiguously_represented_in_contiguous_memory_region(),
+                .default_field = a,
+        };
+        DataTypePtr arr = remove_nullable(type);
+        meta_info_assert(arr, arr_meta_info_to_assert);
+    }
+}
+
+TEST_F(DataTypeArrayTest, CreateColumnTest) {
+    // for tinyint -> largeint, float,double, ipv4/6, all date type  here uncompressed size is 16
+    // for string here uncompressed size is 24
+    for (int i = 0; i < 13; i++) {
+        auto type = remove_nullable(array_types[i]);
+        // any different nested type in arr with same default array ?
+        Field default_field_array = Array();
+        create_column_assert(type, default_field_array, 16);
+    }
+    {
+        auto type = remove_nullable(array_types[13]);
+        Field default_field_array = Array();
+        create_column_assert(type, default_field_array, 24);
+    }
+    // for decimal32/64/128/256 here uncompressed size is 16
+    // one scalar type
+    for (int i = 14; i < 18; i++) {
+        auto type = remove_nullable(array_types[i]);
+        Field default_field_array = Array();
+        create_column_assert(type, default_field_array, 16);
+    }
+    // for array-array-scala
+    for (int i = 18; i < 31; i++) {
+        auto type = remove_nullable(array_types[i]);
+        Field default_field_array = Array();
+        create_column_assert(type, default_field_array, 28);
+    }
+    {
+        // string type
+        auto type = remove_nullable(array_types[31]);
+        Field default_field_array = Array();
+        create_column_assert(type, default_field_array, 36);
+    }
+    for (int i = 32; i < 36; i++) {
+        auto type = remove_nullable(array_types[i]);
+        Field default_field_array = Array();
+        create_column_assert(type, default_field_array, 28);
+    }
+    // for array-map
+    {
+        auto type = remove_nullable(array_types[36]);
+        Field default_field_array = Array();
+        create_column_assert(type, default_field_array, 44);
+        type = remove_nullable(array_types[39]);
+        default_field_array = Array();
+        create_column_assert(type, default_field_array, 44);
+    }
+    {
+        auto type = remove_nullable(array_types[37]);
+        Field default_field_array = Array();
+        create_column_assert(type, default_field_array, 36);
+        type = remove_nullable(array_types[38]);
+        default_field_array = Array();
+        create_column_assert(type, default_field_array, 36);
+    }
+    // for array-struct
+    {
+        auto type = remove_nullable(array_types[40]);
+        Field default_field_array = Array();
+        create_column_assert(type, default_field_array, 76);
+    }
+}
+
+TEST_F(DataTypeArrayTest, GetFieldTest) {
+    TExprNode node;
+    node.node_type = TExprNodeType::ARRAY_LITERAL;
+    for (size_t i = 0; i < array_columns.size(); ++i) {
+        Field assert_field;
+        array_columns[i]->get(0, assert_field);
+        get_field_assert(array_types[i], node, assert_field, true);
+    }
+}
+
+TEST_F(DataTypeArrayTest, FromAndToStringTest) {
+    // insert from data csv and assert insert result
+    for (int i = 0; i < array_columns.size(); i++) {
+        auto& column = array_columns[i];
+        auto& type = array_types[i];
+        std::cout << "type: " << type->get_name() << " for column size: " << column->size()
+                  << std::endl;
+        // datatype array<string> has some different behavior maybe wrong with given data
+        if (i == 13 || i == 31) {
+            continue;
+        }
+        assert_to_string_from_string_assert(column->assume_mutable(), type);
+    }
+}
+
+TEST_F(DataTypeArrayTest, CompareTest) {
+    for (int i = 0; i < array_types.size(); i++) {
+        auto& type = array_types[i];
+        auto array_type = assert_cast<const DataTypeArray*>(remove_nullable(type).get());
+        auto nested_type = array_type->get_nested_type();
+        EXPECT_EQ(nested_type->is_comparable(), array_type->is_comparable());
+        EXPECT_FALSE(array_type->equals(*type));
+    }
+}
+
+TEST_F(DataTypeArrayTest, SerdeHiveTextAndJsonFormatTest) {
+    // insert from data csv and assert insert result
+    for (int i = 0; i < 40; i++) {
+        MutableColumns array_cols;
+        array_cols.push_back(array_columns[i]->get_ptr());
+        // array-struct would cause be core:heap-buffer-overflow for hive_text deser as '[]'
+        CommonDataTypeSerdeTest::load_data_and_assert_from_csv<true, true>({serdes[i]}, array_cols,
+                                                                           data_files[i], ';');
+        CommonDataTypeSerdeTest::load_data_and_assert_from_csv<false, true>({serdes[i]}, array_cols,
+                                                                            data_files[i], ';');
+    }
+}
+
+TEST_F(DataTypeArrayTest, SerdePbTest) {
+    // fix serde pb for read decimal64 not support
+    MutableColumns array_cols;
+    DataTypeSerDeSPtrs serdes_pb;
+    for (int i = 0; i < 40; i++) {
+        array_cols.push_back(array_columns[i]->get_ptr());
+        serdes_pb.push_back(serdes[i]);
+    }
+    CommonDataTypeSerdeTest::assert_pb_format(array_cols, serdes_pb);
+}
+
+TEST_F(DataTypeArrayTest, SerdeJsonbTest) {
+    CommonDataTypeSerdeTest::assert_jsonb_format(array_columns, serdes);
+}
+
+TEST_F(DataTypeArrayTest, SerdeMysqlTest) {
+    // insert from data csv and assert insert result
+    CommonDataTypeSerdeTest::assert_mysql_format(array_columns, serdes);
+}
+
+TEST_F(DataTypeArrayTest, SerializeDeserializeTest) {
+    // insert from data csv and assert insert result
+    CommonDataTypeTest::serialize_deserialize_assert(array_columns, array_types);
+}
+
+TEST_F(DataTypeArrayTest, SerdeArrowTest) {
+    // todo. fix decimal256 serde
+    MutableColumns array_cols;
+    for (int i = 0; i < 17; i++) {
+        array_cols.push_back(array_columns[i]->get_ptr());
+    }
+    for (int i = 18; i < 35; i++) {
+        array_cols.push_back(array_columns[i]->get_ptr());
+    }
+    array_cols.push_back(array_columns[36]->get_ptr());
+    DataTypes types;
+    for (int i = 0; i < 17; i++) {
+        types.push_back(array_types[i]);
+    }
+    for (int i = 18; i < 35; i++) {
+        types.push_back(array_types[i]);
+    }
+    types.push_back(array_types[36]);
+    DataTypeSerDeSPtrs serde;
+    for (int i = 0; i < 17; i++) {
+        serde.push_back(serdes[i]);
+    }
+    for (int i = 18; i < 35; i++) {
+        serde.push_back(serdes[i]);
+    }
+    serde.push_back(serdes[36]);
+    CommonDataTypeSerdeTest::assert_arrow_format(array_cols, serde, types);
+    {
+        for (int i = 38; i < 41; ++i) {
+            MutableColumns error_cols;
+            error_cols.push_back(array_columns[i]->get_ptr());
+            DataTypeSerDeSPtrs serde1;
+            serde1.push_back(serdes[i]);
+            DataTypes typ;
+            typ.push_back(array_types[i]);
+            EXPECT_ANY_THROW(CommonDataTypeSerdeTest::assert_arrow_format(error_cols, serde1, typ));
+        }
+    }
+}
+
+//================== datatype for array ut test ==================
+TEST_F(DataTypeArrayTest, GetNumberOfDimensionsTest) {
+    // for array-scalar
+    for (int i = 0; i < 18; i++) {
+        auto& type = array_types[i];
+        auto array_type = assert_cast<const DataTypeArray*>(remove_nullable(type).get());
+        // array dimension is only for array to nested array , if array nested map or struct, the dimension also be is 1
+        EXPECT_EQ(array_type->get_number_of_dimensions(), 1) << "for type: " << type->get_name();
+    }
+    // for array-array
+    for (int i = 18; i < 36; i++) {
+        auto& type = array_types[i];
+        auto desc = array_descs[i];
+        auto array_type = assert_cast<const DataTypeArray*>(remove_nullable(type).get());
+        // array dimension is only for array to nested array , if array nested map or struct, the dimension also be is 1
+        EXPECT_EQ(array_type->get_number_of_dimensions(), 2) << "for type: " << type->get_name();
+    }
+    for (int i = 36; i < 41; i++) {
+        auto& type = array_types[i];
+        auto array_type = assert_cast<const DataTypeArray*>(remove_nullable(type).get());
+        // array dimension is only for array to nested array , if array nested map or struct, the dimension also be is 1
+        EXPECT_EQ(array_type->get_number_of_dimensions(), 1) << "for type: " << type->get_name();
+    }
+}
+
+} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/test/vec/data_types/data_type_ip_test.cpp b/be/test/vec/data_types/data_type_ip_test.cpp
index 72a340b9e145b82..c500c7cf2dd416a 100644
--- a/be/test/vec/data_types/data_type_ip_test.cpp
+++ b/be/test/vec/data_types/data_type_ip_test.cpp
@@ -95,8 +95,9 @@ TEST_F(DataTypeIPTest, MetaInfoTest) {
             .scale = size_t(-1),
             .is_null_literal = false,
             .is_value_represented_by_number = true,
-            .pColumnMeta = col_meta.get()
-            //                .is_value_unambiguously_represented_in_contiguous_memory_region = true
+            .pColumnMeta = col_meta.get(),
+            .is_value_unambiguously_represented_in_contiguous_memory_region = true,
+            .default_field = UInt64(0),
     };
     TypeDescriptor ipv6_type_descriptor = {PrimitiveType::TYPE_IPV6};
     auto col_meta6 = std::make_shared<PColumnMeta>();
@@ -115,9 +116,9 @@ TEST_F(DataTypeIPTest, MetaInfoTest) {
             .scale = size_t(-1),
             .is_null_literal = false,
             .is_value_represented_by_number = true,
-            .pColumnMeta = col_meta6.get()
-            //                .is_value_unambiguously_represented_in_contiguous_memory_region = true
-    };
+            .pColumnMeta = col_meta6.get(),
+            .is_value_unambiguously_represented_in_contiguous_memory_region = true,
+            .default_field = IPv6(0)};
     meta_info_assert(dt_ipv4, ipv4_meta_info_to_assert);
     meta_info_assert(dt_ipv6, ipv6_meta_info);
 }
@@ -125,8 +126,8 @@ TEST_F(DataTypeIPTest, MetaInfoTest) {
 TEST_F(DataTypeIPTest, CreateColumnTest) {
     Field default_field_ipv4 = IPv4(0);
     Field default_field_ipv6 = IPv6(0);
-    create_column_assert(dt_ipv4, default_field_ipv4);
-    create_column_assert(dt_ipv6, default_field_ipv6);
+    create_column_assert(dt_ipv4, default_field_ipv4, 4);
+    create_column_assert(dt_ipv6, default_field_ipv6, 4);
 }
 
 TEST_F(DataTypeIPTest, GetFieldTest) {
@@ -247,4 +248,21 @@ TEST_F(DataTypeIPTest, SerdeJsonbTest) {
                                         CommonDataTypeSerdeTest::assert_jsonb_format);
 }
 
+TEST_F(DataTypeIPTest, SerdeMysqlAndArrowTest) {
+    auto serde_ipv4 = dt_ipv4->get_serde(1);
+    auto serde_ipv6 = dt_ipv6->get_serde(1);
+    auto column_ipv4 = dt_ipv4->create_column();
+    auto column_ipv6 = dt_ipv6->create_column();
+
+    // insert from data csv and assert insert result
+    MutableColumns ip_cols;
+    ip_cols.push_back(column_ipv4->get_ptr());
+    ip_cols.push_back(column_ipv6->get_ptr());
+    DataTypeSerDeSPtrs serde = {dt_ipv4->get_serde(), dt_ipv6->get_serde()};
+    CommonDataTypeSerdeTest::check_data(ip_cols, serde, ';', {1, 2}, data_files[0],
+                                        CommonDataTypeSerdeTest::assert_mysql_format);
+
+    CommonDataTypeSerdeTest::assert_arrow_format(ip_cols, serde, {dt_ipv4, dt_ipv6});
+}
+
 } // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/test/vec/function/function_test_util.cpp b/be/test/vec/function/function_test_util.cpp
index ddc97c440ed07af..287b5d5c4b3b29b 100644
--- a/be/test/vec/function/function_test_util.cpp
+++ b/be/test/vec/function/function_test_util.cpp
@@ -34,7 +34,9 @@
 #include "vec/data_types/data_type_ipv4.h"
 #include "vec/data_types/data_type_ipv6.h"
 #include "vec/data_types/data_type_jsonb.h"
+#include "vec/data_types/data_type_map.h"
 #include "vec/data_types/data_type_string.h"
+#include "vec/data_types/data_type_struct.h"
 #include "vec/data_types/data_type_time_v2.h"
 #include "vec/exprs/table_function/table_function.h"
 #include "vec/runtime/vdatetime_value.h"
@@ -189,9 +191,61 @@ size_t type_index_to_data_type(const std::vector<AnyType>& input_types, size_t i
             return ret;
         }
         desc.children.push_back(sub_desc.type_desc);
+        if (sub_desc.is_nullable) {
+            sub_type = make_nullable(sub_type);
+        }
         type = std::make_shared<DataTypeArray>(sub_type);
         return ret + 1;
     }
+    case TypeIndex::Map: {
+        desc.type = doris::PrimitiveType::TYPE_MAP;
+        ut_type::UTDataTypeDesc key_desc;
+        DataTypePtr key_type = nullptr;
+        ut_type::UTDataTypeDesc value_desc;
+        DataTypePtr value_type = nullptr;
+        ++index;
+        size_t ret = type_index_to_data_type(input_types, index, key_desc, key_type);
+        if (ret <= 0) {
+            return ret;
+        }
+        ++index;
+        ret = type_index_to_data_type(input_types, index, value_desc, value_type);
+        if (ret <= 0) {
+            return ret;
+        }
+        desc.children.push_back(key_desc.type_desc);
+        desc.children.push_back(value_desc.type_desc);
+        if (key_desc.is_nullable) {
+            key_type = make_nullable(key_type);
+        }
+        if (value_desc.is_nullable) {
+            value_type = make_nullable(value_type);
+        }
+        type = std::make_shared<DataTypeMap>(key_type, value_type);
+        return ret + 1;
+    }
+    case TypeIndex::Struct: {
+        desc.type = doris::PrimitiveType::TYPE_STRUCT;
+        ++index;
+        size_t ret = 0;
+        DataTypes sub_types;
+        while (index < input_types.size()) {
+            ut_type::UTDataTypeDesc sub_desc;
+            DataTypePtr sub_type = nullptr;
+            ret = type_index_to_data_type(input_types, index, sub_desc, sub_type);
+            if (ret <= 0) {
+                return ret;
+            }
+            desc.children.push_back(sub_desc.type_desc);
+            if (sub_desc.is_nullable) {
+                sub_type = make_nullable(sub_type);
+                sub_types.push_back(sub_type);
+            }
+            ++index;
+        }
+        type = std::make_shared<DataTypeStruct>(sub_types);
+        return ret + 1;
+    }
     case TypeIndex::Nullable: {
         ++index;
         size_t ret = type_index_to_data_type(input_types, index, ut_desc, type);
@@ -220,6 +274,9 @@ bool parse_ut_data_type(const std::vector<AnyType>& input_types, ut_type::UTData
         }
         size_t res = type_index_to_data_type(input_types, i, desc, desc.data_type);
         if (res <= 0) {
+            std::cout << "return error, res:" << res << ", i:" << i
+                      << ", input_types.size():" << input_types.size()
+                      << "desc : " << desc.type_desc.debug_string() << std::endl;
             return false;
         }
         if (desc.is_nullable) {

From b7f1a41efa7b25b958c55cae0c077338f7dffbfb Mon Sep 17 00:00:00 2001
From: Thearas <gaozifeng@selectdb.com>
Date: Thu, 9 Jan 2025 17:48:33 +0800
Subject: [PATCH 082/140] [opt](oceanbase docker)Use LTS docker image and print
 unhealthy docker logs (#46647)

### What problem does this PR solve?

Problem Summary:
Oceanbase container sometimes start failed.
<img width="653" alt="image"
src="https://github.com/user-attachments/assets/d95c66cf-7e04-4179-a565-9b9dd8b87128"
/>

We do two things:
1. Print last 100 lines docker logs of unhealthy container for debugging
2. Upgrade Oceanbase docker image to the newest `4.2.1-lts`, since it is
7 months newer than `4.2.1`, more stable
---
 .../docker-compose/oceanbase/oceanbase.yaml.tpl           | 2 +-
 docker/thirdparties/run-thirdparties-docker.sh            | 5 +++++
 .../jdbc/test_oceanbase_jdbc_catalog.out                  | 8 ++++----
 3 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/docker/thirdparties/docker-compose/oceanbase/oceanbase.yaml.tpl b/docker/thirdparties/docker-compose/oceanbase/oceanbase.yaml.tpl
index e4f00a6a115e626..d836c1252c14f13 100644
--- a/docker/thirdparties/docker-compose/oceanbase/oceanbase.yaml.tpl
+++ b/docker/thirdparties/docker-compose/oceanbase/oceanbase.yaml.tpl
@@ -19,7 +19,7 @@ version: "2.1"
 
 services:
   doris--oceanbase:
-    image: oceanbase/oceanbase-ce:4.2.1
+    image: oceanbase/oceanbase-ce:4.2.1-lts
     restart: always
     environment:
       MODE: slim
diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh
index 16c8b9e7d6b5505..fd9558eef558b06 100755
--- a/docker/thirdparties/run-thirdparties-docker.sh
+++ b/docker/thirdparties/run-thirdparties-docker.sh
@@ -718,6 +718,11 @@ for compose in "${!pids[@]}"; do
         echo "docker $compose started failed with status $status"
         echo "print start_${compose}.log"
         cat start_${compose}.log
+
+        echo ""
+        echo "print last 100 logs of the latest unhealthy container"
+        docker ps -a --latest --filter 'health=unhealthy' --format '{{.ID}}' | xargs -I '{}' sh -c 'echo "=== Logs of {} ===" && docker logs -t --tail 100 "{}"'
+
         exit 1
     fi
 done
diff --git a/regression-test/data/external_table_p0/jdbc/test_oceanbase_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_oceanbase_jdbc_catalog.out
index fdf887af2a2a4eb..8d8b5d7b0f9a8be 100644
--- a/regression-test/data/external_table_p0/jdbc/test_oceanbase_jdbc_catalog.out
+++ b/regression-test/data/external_table_p0/jdbc/test_oceanbase_jdbc_catalog.out
@@ -1,7 +1,7 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !query --
-\N	302	\N	502	602	4.14159	\N	6.141592	\N	-124	-302	2013	-402	-502	-602	\N	2012-10-26T02:08:39.345700	2013-10-26T08:09:18	-5.14145	\N	-7.1400	row2	\N	09:11:09.567	text2	0xE86F6C6C6F20576F726C67	\N	\N	0x2F	\N	0x88656C6C9F	Value3
-201	301	401	501	601	3.14159	4.1415926	5.141592	1	-123	-301	2012	-401	-501	-601	2012-10-30	2012-10-25T12:05:36.345700	2012-10-25T08:08:08	-4.14145	-5.1400000001	-6.1400	row1	line1	09:09:09.567	text1	0x48656C6C6F20576F726C64	{"age": 30, "city": "London", "name": "Alice"}	Option1,Option3	0x2A	0x48656C6C6F00000000000000	0x48656C6C6F	Value2
-202	302	402	502	602	4.14159	5.1415926	6.141592	0	-124	-302	2013	-402	-502	-602	2012-11-01	2012-10-26T02:08:39.345700	2013-10-26T08:09:18	-5.14145	-6.1400000001	-7.1400	row2	line2	09:11:09.567	text2	0xE86F6C6C6F20576F726C67	{"age": 18, "city": "ChongQing", "name": "Gaoxin"}	Option1,Option2	0x2F	0x58676C6C6F00000000000000	0x88656C6C9F	Value3
-203	303	403	503	603	7.14159	8.1415926	9.141592	0	\N	-402	2017	-602	-902	-1102	2012-11-02	\N	2013-10-27T08:11:18	-5.14145	-6.1400000000001	-7.1400	row3	line3	09:11:09.567	text3	0xE86F6C6C6F20576F726C67	{"age": 24, "city": "ChongQing", "name": "ChenQi"}	Option2	0x2F	0x58676C6C6F00000000000000	\N	Value1
+\N	302	\N	502	602	4.14159	\N	6.141592	\N	-124	-302	2013	-402	-502	-602	\N	2012-10-26T10:08:39.345700	2013-10-26T16:09:18	-5.14145	\N	-7.1400	row2	\N	09:11:09.567	text2	0xE86F6C6C6F20576F726C67	\N	\N	0x2F	\N	0x88656C6C9F	Value3
+201	301	401	501	601	3.14159	4.1415926	5.141592	1	-123	-301	2012	-401	-501	-601	2012-10-30	2012-10-25T20:05:36.345700	2012-10-25T16:08:08	-4.14145	-5.1400000001	-6.1400	row1	line1	09:09:09.567	text1	0x48656C6C6F20576F726C64	{"age": 30, "city": "London", "name": "Alice"}	Option1,Option3	0x2A	0x48656C6C6F00000000000000	0x48656C6C6F	Value2
+202	302	402	502	602	4.14159	5.1415926	6.141592	0	-124	-302	2013	-402	-502	-602	2012-11-01	2012-10-26T10:08:39.345700	2013-10-26T16:09:18	-5.14145	-6.1400000001	-7.1400	row2	line2	09:11:09.567	text2	0xE86F6C6C6F20576F726C67	{"age": 18, "city": "ChongQing", "name": "Gaoxin"}	Option1,Option2	0x2F	0x58676C6C6F00000000000000	0x88656C6C9F	Value3
+203	303	403	503	603	7.14159	8.1415926	9.141592	0	\N	-402	2017	-602	-902	-1102	2012-11-02	\N	2013-10-27T16:11:18	-5.14145	-6.1400000000001	-7.1400	row3	line3	09:11:09.567	text3	0xE86F6C6C6F20576F726C67	{"age": 24, "city": "ChongQing", "name": "ChenQi"}	Option2	0x2F	0x58676C6C6F00000000000000	\N	Value1
 

From 4624dea97d2fa27f062aaaaaba2ee39fbcafd372 Mon Sep 17 00:00:00 2001
From: airborne12 <jiangkai@selectdb.com>
Date: Thu, 9 Jan 2025 18:34:43 +0800
Subject: [PATCH 083/140] [fix](index size) discard index size when meta size
 is invalid (#46549)

### What problem does this PR solve?

Issue Number: close #45817

Related PR: #xxx

Problem Summary:
Try to discard and get real inverted index size when rowset meta index
size is invalid.

### Release note

None

### Check List (For Author)

- Test <!-- At least one of them must be included. -->
    - [ ] Regression test
    - [x] Unit Test
    - [ ] Manual test (add detailed scripts or steps below)
    - [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
        - [ ] Previous test can cover this change.
        - [ ] No code files have been changed.
        - [ ] Other reason <!-- Add your reason?  -->

- Behavior changed:
    - [ ] No.
    - [ ] Yes. <!-- Explain the behavior change -->

- Does this need documentation?
    - [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->

### Check List (For Reviewer who merge this PR)

- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
 be/src/olap/compaction.cpp                    |  22 ++-
 be/src/olap/rowset/beta_rowset.cpp            |   2 +-
 be/src/olap/rowset/beta_rowset.h              |   2 +-
 be/src/olap/rowset/beta_rowset_writer.cpp     |  23 ++-
 be/src/olap/rowset/rowset.h                   |   8 +-
 be/src/olap/rowset/rowset_meta.h              |  12 +-
 be/src/olap/task/index_builder.cpp            |   2 +-
 be/test/olap/ordered_data_compaction_test.cpp | 138 +++++++++++++++++-
 be/test/testutil/mock_rowset.h                |   4 +
 9 files changed, 195 insertions(+), 18 deletions(-)

diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index f4a4960c59a708b..8ef296607eb435c 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -324,9 +324,25 @@ Status CompactionMixin::do_compact_ordered_rowsets() {
 
 void CompactionMixin::build_basic_info() {
     for (auto& rowset : _input_rowsets) {
-        _input_rowsets_data_size += rowset->data_disk_size();
-        _input_rowsets_index_size += rowset->index_disk_size();
-        _input_rowsets_total_size += rowset->total_disk_size();
+        const auto& rowset_meta = rowset->rowset_meta();
+        auto index_size = rowset_meta->index_disk_size();
+        auto total_size = rowset_meta->total_disk_size();
+        auto data_size = rowset_meta->data_disk_size();
+        // corrupted index size caused by bug before 2.1.5 or 3.0.0 version
+        // try to get real index size from disk.
+        if (index_size < 0 || index_size > total_size * 2) {
+            LOG(ERROR) << "invalid index size:" << index_size << " total size:" << total_size
+                       << " data size:" << data_size << " tablet:" << rowset_meta->tablet_id()
+                       << " rowset:" << rowset_meta->rowset_id();
+            index_size = 0;
+            auto st = rowset->get_inverted_index_size(&index_size);
+            if (!st.ok()) {
+                LOG(ERROR) << "failed to get inverted index size. res=" << st;
+            }
+        }
+        _input_rowsets_data_size += data_size;
+        _input_rowsets_index_size += index_size;
+        _input_rowsets_total_size += total_size;
         _input_row_num += rowset->num_rows();
         _input_num_segments += rowset->num_segments();
     }
diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp
index 59019f93c6e7ff2..b25b261c2bbafb1 100644
--- a/be/src/olap/rowset/beta_rowset.cpp
+++ b/be/src/olap/rowset/beta_rowset.cpp
@@ -89,7 +89,7 @@ Status BetaRowset::get_segment_num_rows(std::vector<uint32_t>* segment_rows) {
     return Status::OK();
 }
 
-Status BetaRowset::get_inverted_index_size(size_t* index_size) {
+Status BetaRowset::get_inverted_index_size(int64_t* index_size) {
     const auto& fs = _rowset_meta->fs();
     if (!fs) {
         return Status::Error<INIT_FAILED>("get fs failed, resource_id={}",
diff --git a/be/src/olap/rowset/beta_rowset.h b/be/src/olap/rowset/beta_rowset.h
index c9ed568186a8e9a..4b1388a1f08d038 100644
--- a/be/src/olap/rowset/beta_rowset.h
+++ b/be/src/olap/rowset/beta_rowset.h
@@ -80,7 +80,7 @@ class BetaRowset final : public Rowset {
 
     Status get_segments_size(std::vector<size_t>* segments_size);
 
-    Status get_inverted_index_size(size_t* index_size);
+    Status get_inverted_index_size(int64_t* index_size) override;
 
     [[nodiscard]] virtual Status add_to_binlog() override;
 
diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp b/be/src/olap/rowset/beta_rowset_writer.cpp
index 9207c52c14d10db..8093a9d3bca7a19 100644
--- a/be/src/olap/rowset/beta_rowset_writer.cpp
+++ b/be/src/olap/rowset/beta_rowset_writer.cpp
@@ -652,8 +652,27 @@ Status BaseBetaRowsetWriter::add_rowset(RowsetSharedPtr rowset) {
     assert(rowset->rowset_meta()->rowset_type() == BETA_ROWSET);
     RETURN_IF_ERROR(rowset->link_files_to(_context.tablet_path, _context.rowset_id));
     _num_rows_written += rowset->num_rows();
-    _total_data_size += rowset->rowset_meta()->data_disk_size();
-    _total_index_size += rowset->rowset_meta()->index_disk_size();
+    const auto& rowset_meta = rowset->rowset_meta();
+    auto index_size = rowset_meta->index_disk_size();
+    auto total_size = rowset_meta->total_disk_size();
+    auto data_size = rowset_meta->data_disk_size();
+    // corrupted index size caused by bug before 2.1.5 or 3.0.0 version
+    // try to get real index size from disk.
+    if (index_size < 0 || index_size > total_size * 2) {
+        LOG(ERROR) << "invalid index size:" << index_size << " total size:" << total_size
+                   << " data size:" << data_size << " tablet:" << rowset_meta->tablet_id()
+                   << " rowset:" << rowset_meta->rowset_id();
+        index_size = 0;
+        auto st = rowset->get_inverted_index_size(&index_size);
+        if (!st.ok()) {
+            if (!st.is<NOT_FOUND>()) {
+                LOG(ERROR) << "failed to get inverted index size. res=" << st;
+                return st;
+            }
+        }
+    }
+    _total_data_size += data_size;
+    _total_index_size += index_size;
     _num_segment += cast_set<int32_t>(rowset->num_segments());
     // append key_bounds to current rowset
     RETURN_IF_ERROR(rowset->get_segments_key_bounds(&_segments_encoded_key_bounds));
diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h
index 047a61a39ea22ea..01f321728f1bea9 100644
--- a/be/src/olap/rowset/rowset.h
+++ b/be/src/olap/rowset/rowset.h
@@ -147,9 +147,9 @@ class Rowset : public std::enable_shared_from_this<Rowset>, public MetadataAdder
     // helper class to access RowsetMeta
     int64_t start_version() const { return rowset_meta()->version().first; }
     int64_t end_version() const { return rowset_meta()->version().second; }
-    size_t index_disk_size() const { return rowset_meta()->index_disk_size(); }
-    size_t data_disk_size() const { return rowset_meta()->data_disk_size(); }
-    size_t total_disk_size() const { return rowset_meta()->total_disk_size(); }
+    int64_t index_disk_size() const { return rowset_meta()->index_disk_size(); }
+    int64_t data_disk_size() const { return rowset_meta()->data_disk_size(); }
+    int64_t total_disk_size() const { return rowset_meta()->total_disk_size(); }
     bool empty() const { return rowset_meta()->empty(); }
     bool zero_num_rows() const { return rowset_meta()->num_rows() == 0; }
     size_t num_rows() const { return rowset_meta()->num_rows(); }
@@ -210,6 +210,8 @@ class Rowset : public std::enable_shared_from_this<Rowset>, public MetadataAdder
                                  size_t new_rowset_start_seg_id = 0,
                                  std::set<int64_t>* without_index_uids = nullptr) = 0;
 
+    virtual Status get_inverted_index_size(int64_t* index_size) = 0;
+
     // copy all files to `dir`
     virtual Status copy_files_to(const std::string& dir, const RowsetId& new_rowset_id) = 0;
 
diff --git a/be/src/olap/rowset/rowset_meta.h b/be/src/olap/rowset/rowset_meta.h
index 46121aeae2be6de..40e55e074528cde 100644
--- a/be/src/olap/rowset/rowset_meta.h
+++ b/be/src/olap/rowset/rowset_meta.h
@@ -130,21 +130,21 @@ class RowsetMeta : public MetadataAdder<RowsetMeta> {
 
     void set_num_rows(int64_t num_rows) { _rowset_meta_pb.set_num_rows(num_rows); }
 
-    size_t total_disk_size() const { return _rowset_meta_pb.total_disk_size(); }
+    int64_t total_disk_size() const { return _rowset_meta_pb.total_disk_size(); }
 
-    void set_total_disk_size(size_t total_disk_size) {
+    void set_total_disk_size(int64_t total_disk_size) {
         _rowset_meta_pb.set_total_disk_size(total_disk_size);
     }
 
-    size_t data_disk_size() const { return _rowset_meta_pb.data_disk_size(); }
+    int64_t data_disk_size() const { return _rowset_meta_pb.data_disk_size(); }
 
-    void set_data_disk_size(size_t data_disk_size) {
+    void set_data_disk_size(int64_t data_disk_size) {
         _rowset_meta_pb.set_data_disk_size(data_disk_size);
     }
 
-    size_t index_disk_size() const { return _rowset_meta_pb.index_disk_size(); }
+    int64_t index_disk_size() const { return _rowset_meta_pb.index_disk_size(); }
 
-    void set_index_disk_size(size_t index_disk_size) {
+    void set_index_disk_size(int64_t index_disk_size) {
         _rowset_meta_pb.set_index_disk_size(index_disk_size);
     }
 
diff --git a/be/src/olap/task/index_builder.cpp b/be/src/olap/task/index_builder.cpp
index bd17dedbbeeffb6..bc677ea6f5c0492 100644
--- a/be/src/olap/task/index_builder.cpp
+++ b/be/src/olap/task/index_builder.cpp
@@ -81,7 +81,7 @@ Status IndexBuilder::update_inverted_index_info() {
         TabletSchemaSPtr output_rs_tablet_schema = std::make_shared<TabletSchema>();
         const auto& input_rs_tablet_schema = input_rowset->tablet_schema();
         output_rs_tablet_schema->copy_from(*input_rs_tablet_schema);
-        size_t total_index_size = 0;
+        int64_t total_index_size = 0;
         auto* beta_rowset = reinterpret_cast<BetaRowset*>(input_rowset.get());
         auto size_st = beta_rowset->get_inverted_index_size(&total_index_size);
         DBUG_EXECUTE_IF("IndexBuilder::update_inverted_index_info_size_st_not_ok", {
diff --git a/be/test/olap/ordered_data_compaction_test.cpp b/be/test/olap/ordered_data_compaction_test.cpp
index 831e3031378e4e5..dbbe19fc475e58f 100644
--- a/be/test/olap/ordered_data_compaction_test.cpp
+++ b/be/test/olap/ordered_data_compaction_test.cpp
@@ -89,6 +89,15 @@ class OrderedDataCompactionTest : public ::testing::Test {
         EXPECT_TRUE(io::global_local_filesystem()
                             ->create_directory(absolute_dir + "/tablet_path")
                             .ok());
+        // tmp dir
+        EXPECT_TRUE(io::global_local_filesystem()->delete_directory(tmp_dir).ok());
+        EXPECT_TRUE(io::global_local_filesystem()->create_directory(tmp_dir).ok());
+        std::vector<StorePath> paths;
+        paths.emplace_back(std::string(tmp_dir), 1024000000);
+        auto tmp_file_dirs = std::make_unique<segment_v2::TmpFileDirs>(paths);
+        st = tmp_file_dirs->init();
+        EXPECT_TRUE(st.ok()) << st.to_json();
+        ExecEnv::GetInstance()->set_tmp_file_dir(std::move(tmp_file_dirs));
 
         doris::EngineOptions options;
         auto engine = std::make_unique<StorageEngine>(options);
@@ -153,6 +162,62 @@ class OrderedDataCompactionTest : public ::testing::Test {
         return tablet_schema;
     }
 
+    TabletSchemaSPtr create_inverted_index_v1_schema(KeysType keys_type = DUP_KEYS) {
+        TabletSchemaSPtr tablet_schema = std::make_shared<TabletSchema>();
+        TabletSchemaPB tablet_schema_pb;
+        tablet_schema_pb.set_keys_type(keys_type);
+        tablet_schema_pb.set_num_short_key_columns(1);
+        tablet_schema_pb.set_num_rows_per_row_block(1024);
+        tablet_schema_pb.set_compress_kind(COMPRESS_NONE);
+        tablet_schema_pb.set_next_column_unique_id(4);
+        tablet_schema_pb.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V1);
+
+        auto* index_pb = tablet_schema_pb.add_index();
+        index_pb->set_index_id(1);
+        index_pb->set_index_name("c1_index");
+        index_pb->set_index_type(IndexType::INVERTED);
+        index_pb->add_col_unique_id(2);
+
+        ColumnPB* column_1 = tablet_schema_pb.add_column();
+        column_1->set_unique_id(1);
+        column_1->set_name("c1");
+        column_1->set_type("INT");
+        column_1->set_is_key(true);
+        column_1->set_length(4);
+        column_1->set_index_length(4);
+        column_1->set_is_nullable(false);
+        column_1->set_is_bf_column(false);
+
+        ColumnPB* column_2 = tablet_schema_pb.add_column();
+        column_2->set_unique_id(2);
+        column_2->set_name("c2");
+        column_2->set_type("INT");
+        column_2->set_length(4);
+        column_2->set_index_length(4);
+        column_2->set_is_nullable(true);
+        column_2->set_is_key(false);
+        column_2->set_is_nullable(false);
+        column_2->set_is_bf_column(false);
+
+        // unique table must contains the DELETE_SIGN column
+        if (keys_type == UNIQUE_KEYS) {
+            ColumnPB* column_3 = tablet_schema_pb.add_column();
+            column_3->set_unique_id(3);
+            column_3->set_name(DELETE_SIGN);
+            column_3->set_type("TINYINT");
+            column_3->set_length(1);
+            column_3->set_index_length(1);
+            column_3->set_is_nullable(false);
+            column_3->set_is_key(false);
+            column_3->set_is_nullable(false);
+            column_3->set_is_bf_column(false);
+        }
+
+        tablet_schema->init_from_pb(tablet_schema_pb);
+
+        return tablet_schema;
+    }
+
     TabletSchemaSPtr create_agg_schema() {
         TabletSchemaSPtr tablet_schema = std::make_shared<TabletSchema>();
         TabletSchemaPB tablet_schema_pb;
@@ -401,7 +466,8 @@ class OrderedDataCompactionTest : public ::testing::Test {
     }
 
 private:
-    const std::string kTestDir = "/ut_dir/vertical_compaction_test";
+    const std::string kTestDir = "/ut_dir/ordered_compaction_test";
+    const std::string tmp_dir = "./ut_dir/ordered_compaction_test/tmp";
     string absolute_dir;
     std::unique_ptr<DataDir> _data_dir;
 };
@@ -485,5 +551,75 @@ TEST_F(OrderedDataCompactionTest, test_01) {
     }
 }
 
+TEST_F(OrderedDataCompactionTest, test_index_disk_size) {
+    auto num_input_rowset = 3;
+    auto num_segments = 2;
+    auto rows_per_segment = 50;
+    std::vector<std::vector<std::vector<std::tuple<int64_t, int64_t>>>> input_data;
+    generate_input_data(num_input_rowset, num_segments, rows_per_segment, input_data);
+
+    TabletSchemaSPtr tablet_schema = create_inverted_index_v1_schema();
+    TabletSharedPtr tablet = create_tablet(*tablet_schema, false, 10000, false);
+    EXPECT_TRUE(io::global_local_filesystem()->create_directory(tablet->tablet_path()).ok());
+
+    vector<RowsetSharedPtr> input_rowsets;
+    SegmentsOverlapPB new_overlap = NONOVERLAPPING;
+    for (auto i = 0; i < num_input_rowset; i++) {
+        RowsetWriterContext writer_context;
+        create_rowset_writer_context(tablet_schema, tablet->tablet_path(), new_overlap, UINT32_MAX,
+                                     &writer_context);
+
+        auto res = RowsetFactory::create_rowset_writer(*engine_ref, writer_context, false);
+        EXPECT_TRUE(res.has_value()) << res.error();
+        auto rowset_writer = std::move(res).value();
+
+        uint32_t num_rows = 0;
+        for (int j = 0; j < input_data[i].size(); ++j) {
+            vectorized::Block block = tablet_schema->create_block();
+            auto columns = block.mutate_columns();
+            for (int rid = 0; rid < input_data[i][j].size(); ++rid) {
+                int32_t c1 = std::get<0>(input_data[i][j][rid]);
+                int32_t c2 = std::get<1>(input_data[i][j][rid]);
+                columns[0]->insert_data((const char*)&c1, sizeof(c1));
+                columns[1]->insert_data((const char*)&c2, sizeof(c2));
+
+                if (tablet_schema->keys_type() == UNIQUE_KEYS) {
+                    uint8_t num = 0;
+                    columns[2]->insert_data((const char*)&num, sizeof(num));
+                }
+                num_rows++;
+            }
+            auto s = rowset_writer->add_block(&block);
+            EXPECT_TRUE(s.ok());
+            s = rowset_writer->flush();
+            EXPECT_TRUE(s.ok());
+        }
+
+        RowsetSharedPtr rowset;
+        EXPECT_EQ(Status::OK(), rowset_writer->build(rowset));
+        EXPECT_EQ(input_data[i].size(), rowset->rowset_meta()->num_segments());
+        EXPECT_EQ(num_rows, rowset->rowset_meta()->num_rows());
+
+        // Set random index_disk_size
+        rowset->rowset_meta()->set_index_disk_size(1024000000000000LL);
+        input_rowsets.push_back(rowset);
+    }
+
+    CumulativeCompaction cu_compaction(*engine_ref, tablet);
+    cu_compaction._input_rowsets = std::move(input_rowsets);
+    EXPECT_EQ(cu_compaction.handle_ordered_data_compaction(), true);
+
+    auto& out_rowset = cu_compaction._output_rowset;
+
+    // Verify the index_disk_size of the output rowset
+    int64_t expected_total_size = 0;
+    for (const auto& rowset : cu_compaction._input_rowsets) {
+        expected_total_size += rowset->rowset_meta()->total_disk_size();
+    }
+    std::cout << "expected_total_size: " << expected_total_size << std::endl;
+    std::cout << "actual_total_disk_size: " << out_rowset->rowset_meta()->total_disk_size()
+              << std::endl;
+    EXPECT_EQ(out_rowset->rowset_meta()->total_disk_size(), expected_total_size);
+}
 } // namespace vectorized
 } // namespace doris
diff --git a/be/test/testutil/mock_rowset.h b/be/test/testutil/mock_rowset.h
index f1472b435b5ac2a..78f8f71bee844fb 100644
--- a/be/test/testutil/mock_rowset.h
+++ b/be/test/testutil/mock_rowset.h
@@ -50,6 +50,10 @@ class MockRowset : public Rowset {
         return Status::NotSupported("MockRowset not support this method.");
     }
 
+    Status get_inverted_index_size(int64_t* index_size) override {
+        return Status::NotSupported("MockRowset not support this method.");
+    }
+
     void clear_inverted_index_cache() override {}
 
     Status get_segments_key_bounds(std::vector<KeyBoundsPB>* segments_key_bounds) override {

From dce5b01f3eec2187f3725e2fd2b20316daab99fd Mon Sep 17 00:00:00 2001
From: morrySnow <zhangwenxin@selectdb.com>
Date: Thu, 9 Jan 2025 18:41:50 +0800
Subject: [PATCH 084/140] [fix](test) eager agg test use be num as 1 for stable
 shape (#46558)

### What problem does this PR solve?
---
 .../push_down_aggr_distinct_through_join_one_side_cust.groovy   | 1 +
 .../push_down_count_distinct_through_join_one_side.groovy       | 2 ++
 .../eager_aggregate/push_down_count_through_join.groovy         | 1 +
 .../push_down_count_through_join_one_side.groovy                | 1 +
 .../eager_aggregate/push_down_max_through_join.groovy           | 1 +
 .../push_down_min_distinct_through_join_one_side.groovy         | 1 +
 .../eager_aggregate/push_down_min_through_join.groovy           | 1 +
 .../push_down_sum_distinct_through_join_one_side.groovy         | 1 +
 .../eager_aggregate/push_down_sum_through_join.groovy           | 1 +
 .../eager_aggregate/push_down_sum_through_join_one_side.groovy  | 1 +
 10 files changed, 11 insertions(+)

diff --git a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_aggr_distinct_through_join_one_side_cust.groovy b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_aggr_distinct_through_join_one_side_cust.groovy
index cc75e289a605fc1..0dbe0cd9c725084 100644
--- a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_aggr_distinct_through_join_one_side_cust.groovy
+++ b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_aggr_distinct_through_join_one_side_cust.groovy
@@ -19,6 +19,7 @@ suite("push_down_aggr_distinct_through_join_one_side_cust") {
     sql "SET enable_nereids_planner=true"
     sql "set runtime_filter_mode=OFF"
     sql "SET enable_fallback_to_original_planner=false"
+    sql "set be_number_for_test=1"
     sql "set DISABLE_NEREIDS_RULES='PRUNE_EMPTY_PARTITION, ELIMINATE_GROUP_BY_KEY_BY_UNIFORM'"
 
     sql """
diff --git a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_count_distinct_through_join_one_side.groovy b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_count_distinct_through_join_one_side.groovy
index 65a6b15ceccf735..26d8a78512ff02e 100644
--- a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_count_distinct_through_join_one_side.groovy
+++ b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_count_distinct_through_join_one_side.groovy
@@ -19,6 +19,8 @@ suite("push_down_count_distinct_through_join_one_side") {
     sql "SET enable_nereids_planner=true"
     sql "set runtime_filter_mode=OFF"
     sql "SET enable_fallback_to_original_planner=false"
+    sql "set be_number_for_test=1"
+
     sql """
         DROP TABLE IF EXISTS count_with_distinct_t;
     """
diff --git a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_count_through_join.groovy b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_count_through_join.groovy
index 7a60b34c54f3a18..bcb989d1c956cc1 100644
--- a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_count_through_join.groovy
+++ b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_count_through_join.groovy
@@ -20,6 +20,7 @@ suite("push_down_count_through_join") {
     sql "set runtime_filter_mode=OFF"
     sql "SET enable_fallback_to_original_planner=false"
     sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'"
+    sql "set be_number_for_test=1"
     sql "set DISABLE_NEREIDS_RULES='ONE_PHASE_AGGREGATE_WITHOUT_DISTINCT, ONE_PHASE_AGGREGATE_SINGLE_DISTINCT_TO_MULTI'"
 
     sql """
diff --git a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_count_through_join_one_side.groovy b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_count_through_join_one_side.groovy
index 031430a48265ae2..65bd291f67e6dba 100644
--- a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_count_through_join_one_side.groovy
+++ b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_count_through_join_one_side.groovy
@@ -21,6 +21,7 @@ suite("push_down_count_through_join_one_side") {
     sql 'set be_number_for_test=3'
     sql "SET enable_fallback_to_original_planner=false"
     sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'"
+    sql "set be_number_for_test=1"
     sql "set DISABLE_NEREIDS_RULES='ONE_PHASE_AGGREGATE_WITHOUT_DISTINCT, ONE_PHASE_AGGREGATE_SINGLE_DISTINCT_TO_MULTI'"
     sql """
         DROP TABLE IF EXISTS count_t_one_side;
diff --git a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_max_through_join.groovy b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_max_through_join.groovy
index e41f89e116183cb..6e4ccc5f669a374 100644
--- a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_max_through_join.groovy
+++ b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_max_through_join.groovy
@@ -20,6 +20,7 @@ suite("push_down_max_through_join") {
     sql "set runtime_filter_mode=OFF"
     sql "SET enable_fallback_to_original_planner=false"
     sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'"
+    sql "set be_number_for_test=1"
     sql "set DISABLE_NEREIDS_RULES='ONE_PHASE_AGGREGATE_WITHOUT_DISTINCT, ONE_PHASE_AGGREGATE_SINGLE_DISTINCT_TO_MULTI'"
 
     sql """
diff --git a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_min_distinct_through_join_one_side.groovy b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_min_distinct_through_join_one_side.groovy
index 676becbcb6ecfa5..e88dd8cfe13bb50 100644
--- a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_min_distinct_through_join_one_side.groovy
+++ b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_min_distinct_through_join_one_side.groovy
@@ -19,6 +19,7 @@ suite("push_down_min_distinct_through_join_one_side") {
     sql "SET enable_nereids_planner=true"
     sql "set runtime_filter_mode=OFF"
     sql "SET enable_fallback_to_original_planner=false"
+    sql "set be_number_for_test=1"
     sql """
         DROP TABLE IF EXISTS min_with_distinct_t;
     """
diff --git a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_min_through_join.groovy b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_min_through_join.groovy
index 99602a9deae3afa..fb728057f577509 100644
--- a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_min_through_join.groovy
+++ b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_min_through_join.groovy
@@ -20,6 +20,7 @@ suite("push_down_min_through_join") {
     sql "set runtime_filter_mode=OFF"
     sql "SET enable_fallback_to_original_planner=false"
     sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'"
+    sql "set be_number_for_test=1"
     sql "set DISABLE_NEREIDS_RULES='ONE_PHASE_AGGREGATE_WITHOUT_DISTINCT, ONE_PHASE_AGGREGATE_SINGLE_DISTINCT_TO_MULTI'"
 
     sql """
diff --git a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_sum_distinct_through_join_one_side.groovy b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_sum_distinct_through_join_one_side.groovy
index 799d45ba413fb79..560f31bab421190 100644
--- a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_sum_distinct_through_join_one_side.groovy
+++ b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_sum_distinct_through_join_one_side.groovy
@@ -19,6 +19,7 @@ suite("push_down_sum_distinct_through_join_one_side") {
     sql "SET enable_nereids_planner=true"
     sql "set runtime_filter_mode=OFF"
     sql "SET enable_fallback_to_original_planner=false"
+    sql "set be_number_for_test=1"
     sql """
         DROP TABLE IF EXISTS sum_with_distinct_t;
     """
diff --git a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_sum_through_join.groovy b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_sum_through_join.groovy
index d6248c79fd7ab69..844b07807f4680d 100644
--- a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_sum_through_join.groovy
+++ b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_sum_through_join.groovy
@@ -20,6 +20,7 @@ suite("push_down_sum_through_join") {
     sql "set runtime_filter_mode=OFF"
     sql "SET enable_fallback_to_original_planner=false"
     sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'"
+    sql "set be_number_for_test=1"
     sql "set DISABLE_NEREIDS_RULES='ONE_PHASE_AGGREGATE_WITHOUT_DISTINCT, ONE_PHASE_AGGREGATE_SINGLE_DISTINCT_TO_MULTI'"
 
     sql """
diff --git a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_sum_through_join_one_side.groovy b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_sum_through_join_one_side.groovy
index ca75cb699a51c76..4f441d077b48406 100644
--- a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_sum_through_join_one_side.groovy
+++ b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_sum_through_join_one_side.groovy
@@ -20,6 +20,7 @@ suite("push_down_sum_through_join_one_side") {
     sql "set runtime_filter_mode=OFF"
     sql "SET enable_fallback_to_original_planner=false"
     sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'"
+    sql "set be_number_for_test=1"
     sql "set DISABLE_NEREIDS_RULES='ONE_PHASE_AGGREGATE_WITHOUT_DISTINCT, ONE_PHASE_AGGREGATE_SINGLE_DISTINCT_TO_MULTI'"
 
     sql """

From 19554ce15a66ea9062dd8e4add76e122336ae302 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E7=A5=9E=E6=8A=80=E5=9C=88=E5=AD=90?=
 <songguangfan@gmail.com>
Date: Thu, 9 Jan 2025 21:37:22 +0800
Subject: [PATCH 085/140] [fix] Add initialization list to the copy constructor
 of ScanPredicate in jni_connect.h (#46666)

### What problem does this PR solve?

Issue Number: close #xxx

Related PR: #xxx

Problem Summary:
The copy constructor of `ScanPredicate` in `jni_connect.h` does not use
an initialization list, which might result in uninitialized member
variables or unintended behavior.
---
 be/src/vec/exec/jni_connector.h | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/be/src/vec/exec/jni_connector.h b/be/src/vec/exec/jni_connector.h
index e06654dcfe3ca8a..97d0bc7f6aa3e74 100644
--- a/be/src/vec/exec/jni_connector.h
+++ b/be/src/vec/exec/jni_connector.h
@@ -100,13 +100,11 @@ class JniConnector : public ProfileCollector {
 
         ScanPredicate(const std::string column_name) : column_name(std::move(column_name)) {}
 
-        ScanPredicate(const ScanPredicate& other) {
-            column_name = other.column_name;
-            op = other.op;
+        ScanPredicate(const ScanPredicate& other)
+                : column_name(other.column_name), op(other.op), scale(other.scale) {
             for (auto v : other.values) {
                 values.emplace_back(v);
             }
-            scale = other.scale;
         }
 
         int length() {

From 1a99d3c124327cad307726ab833e1bbb9da100df Mon Sep 17 00:00:00 2001
From: zhiqiang <hezhiqiang@selectdb.com>
Date: Thu, 9 Jan 2025 22:12:31 +0800
Subject: [PATCH 086/140] [feat](show profile) impl of show query/load profile
 (#46375)

### What problem does this PR solve?

Implementation of show query/load profile.

Show query/load profile can return profile meta of current frontend.

Usage:
1. When limit clause is not involved, the default return rows is limited
to 20. You can use `show query profile limit 100;` if you need more.
2. Profile path is kept only for syntactic compatibility, it takes no
effect, and is optional.
---
 .../org/apache/doris/nereids/DorisParser.g4   |  4 +-
 .../apache/doris/common/profile/Profile.java  | 10 ++--
 .../doris/common/profile/ProfileManager.java  | 35 +++++++++--
 .../nereids/parser/LogicalPlanBuilder.java    | 29 ++++++++-
 .../commands/ShowLoadProfileCommand.java      | 38 ++++++++----
 .../commands/ShowQueryProfileCommand.java     | 39 ++++++++----
 .../suites/query_profile/show_profile.groovy  | 60 +++++++++++++++++++
 7 files changed, 175 insertions(+), 40 deletions(-)
 create mode 100644 regression-test/suites/query_profile/show_profile.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 7045733bafdba7e..49392e982490cd4 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -255,7 +255,7 @@ supportedShowStatement
     | SHOW ALL? GRANTS                                                              #showGrants
     | SHOW GRANTS FOR userIdentify                                                  #showGrantsForUser
     | SHOW SYNC JOB ((FROM | IN) database=multipartIdentifier)?                     #showSyncJob    
-    | SHOW LOAD PROFILE loadIdPath=STRING_LITERAL                                   #showLoadProfile
+    | SHOW LOAD PROFILE loadIdPath=STRING_LITERAL? limitClause?                     #showLoadProfile
     | SHOW CREATE REPOSITORY FOR identifier                                         #showCreateRepository
     | SHOW VIEW
         (FROM |IN) tableName=multipartIdentifier
@@ -302,7 +302,7 @@ supportedShowStatement
     | SHOW TABLE CREATION ((FROM | IN) database=multipartIdentifier)?
         (LIKE STRING_LITERAL)?                                                      #showTableCreation
     | SHOW TABLET STORAGE FORMAT VERBOSE?                                           #showTabletStorageFormat
-    | SHOW QUERY PROFILE queryIdPath=STRING_LITERAL                                 #showQueryProfile
+    | SHOW QUERY PROFILE queryIdPath=STRING_LITERAL? limitClause?                    #showQueryProfile
     ;
 
 supportedLoadStatement
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java
index 71d51c32faf20d1..a51732474035b41 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java
@@ -81,22 +81,22 @@ public class Profile {
     // profile file name format: time_id
     private static final String SEPERATOR = "_";
 
-    // id will be assgined to id of SummaryProfile.
-    // For broker load, its SummaryPRofile id is a string representation of a long integer,
+    // id will be assigned to id of SummaryProfile.
+    // For broker load, its SummaryProfile id is a string representation of a long integer,
     // for others, it is queryID
     private String id = "";
     // summaryProfile will be serialized to storage as JSON, and we can recover it from storage
     // recover of SummaryProfile is important, because it contains the meta information of the profile
     // we need it to construct memory index for profile retrieving.
     private SummaryProfile summaryProfile = new SummaryProfile();
-    // executionProfiles will be stored to storage as text, when geting profile content, we will read
+    // executionProfiles will be stored to storage as text, when getting profile content, we will read
     // from storage directly.
     private List<ExecutionProfile> executionProfiles = Lists.newArrayList();
     // profileStoragePath will only be assigned when:
     // 1. profile is stored to storage
     // 2. or profile is loaded from storage
     private String profileStoragePath = "";
-    // isQueryFinished means the coordinator or stmtexecutor is finished.
+    // isQueryFinished means the coordinator or stmt executor is finished.
     // does not mean the profile report has finished, since the report is async.
     // finish of collection of profile is marked by isCompleted of ExecutionProfiles.
     private boolean isQueryFinished = false;
@@ -274,7 +274,7 @@ public List<ExecutionProfile> getExecutionProfiles() {
     }
 
     // This API will also add the profile to ProfileManager, so that we could get the profile from ProfileManager.
-    // isFinished ONLY means the coordinator or stmtexecutor is finished.
+    // isFinished ONLY means the coordinator or stmt executor is finished.
     public synchronized void updateSummary(Map<String, String> summaryInfo, boolean isFinished,
             Planner planner) {
         try {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileManager.java
index 981d025792c90c1..ef9f2f7bbaf603f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileManager.java
@@ -120,8 +120,8 @@ public void deleteFromStorage() {
         }
     }
 
-    // this variable is assgiened to true the first time the profile is loaded from storage
-    // no futher write operaiton, so no data race
+    // this variable is assigned to true the first time the profile is loaded from storage
+    // no further write operation, so no data race
     boolean isProfileLoaded = false;
 
     // only protect queryIdDeque; queryIdToProfileMap is concurrent, no need to protect
@@ -129,10 +129,10 @@ public void deleteFromStorage() {
     private ReadLock readLock;
     private WriteLock writeLock;
 
-    // profile id is long string for brocker load
+    // profile id is long string for broker load
     // is TUniqueId for others.
     private Map<String, ProfileElement> queryIdToProfileMap;
-    // Sometimes one Profile is related with multiple execution profiles(Brokerload), so that
+    // Sometimes one Profile is related with multiple execution profiles(Broker-load), so that
     // execution profile's query id is not related with Profile's query id.
     private Map<TUniqueId, ExecutionProfile> queryIdToExecutionProfiles;
 
@@ -151,7 +151,7 @@ public static ProfileManager getInstance() {
         return INSTANCE;
     }
 
-    // The visiablity of ProfileManager() is package level, so that we can write ut for it.
+    // The visibility of ProfileManager() is package level, so that we can write ut for it.
     ProfileManager() {
         super("profile-manager", Config.profile_manager_gc_interval_seconds * 1000);
         lock = new ReentrantReadWriteLock(true);
@@ -942,4 +942,29 @@ private void deleteOutdatedProfilesFromMemory() {
             }
         }
     }
+
+    public List<List<String>> getProfileMetaWithType(ProfileType profileType, long limit) {
+        List<List<String>> result = Lists.newArrayList();
+        readLock.lock();
+
+        try {
+            PriorityQueue<ProfileElement> queueIdDeque = getProfileOrderByQueryFinishTimeDesc();
+            while (!queueIdDeque.isEmpty() && limit > 0) {
+                ProfileElement profileElement = queueIdDeque.poll();
+                Map<String, String> infoStrings = profileElement.infoStrings;
+                if (infoStrings.get(SummaryProfile.TASK_TYPE).equals(profileType.toString())) {
+                    List<String> row = Lists.newArrayList();
+                    for (String str : SummaryProfile.SUMMARY_KEYS) {
+                        row.add(infoStrings.get(str));
+                    }
+                    result.add(row);
+                    limit--;
+                }
+            }
+        } finally {
+            readLock.unlock();
+        }
+
+        return result;
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index b3a62e604c53c45..af267f92bdf5edf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -4320,7 +4320,19 @@ public LogicalPlan visitShowLastInsert(ShowLastInsertContext ctx) {
 
     @Override
     public LogicalPlan visitShowLoadProfile(ShowLoadProfileContext ctx) {
-        return new ShowLoadProfileCommand(ctx.loadIdPath.getText());
+        String loadIdPath = "/"; // default load id path
+        if (ctx.loadIdPath != null) {
+            loadIdPath = stripQuotes(ctx.loadIdPath.getText());
+        }
+
+        long limit = 20;
+        if (ctx.limitClause() != null) {
+            limit = Long.parseLong(ctx.limitClause().limit.getText());
+            if (limit < 0) {
+                throw new ParseException("Limit requires non-negative number, got " + String.valueOf(limit));
+            }
+        }
+        return new ShowLoadProfileCommand(loadIdPath, limit);
     }
 
     @Override
@@ -5384,8 +5396,19 @@ public LogicalPlan visitShowTabletStorageFormat(ShowTabletStorageFormatContext c
 
     @Override
     public LogicalPlan visitShowQueryProfile(ShowQueryProfileContext ctx) {
-        String queryIdPath = stripQuotes(ctx.queryIdPath.getText());
-        return new ShowQueryProfileCommand(queryIdPath);
+        String queryIdPath = "/";
+        if (ctx.queryIdPath != null) {
+            queryIdPath = stripQuotes(ctx.queryIdPath.getText());
+        }
+
+        long limit = 20;
+        if (ctx.limitClause() != null) {
+            limit = Long.parseLong(ctx.limitClause().limit.getText());
+            if (limit < 0) {
+                throw new ParseException("Limit requires non-negative number, got " + String.valueOf(limit));
+            }
+        }
+        return new ShowQueryProfileCommand(queryIdPath, limit);
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowLoadProfileCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowLoadProfileCommand.java
index 5320671ad7794d5..bdbd4fd962ae3d7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowLoadProfileCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowLoadProfileCommand.java
@@ -17,37 +17,51 @@
 
 package org.apache.doris.nereids.trees.plans.commands;
 
-import org.apache.doris.catalog.Env;
-import org.apache.doris.common.Config;
-import org.apache.doris.common.UserException;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.profile.ProfileManager;
+import org.apache.doris.common.profile.ProfileManager.ProfileType;
+import org.apache.doris.common.profile.SummaryProfile;
 import org.apache.doris.nereids.trees.plans.PlanType;
 import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.ShowResultSetMetaData;
 import org.apache.doris.qe.StmtExecutor;
 
+import java.util.List;
+
 /**
  * show load profile command
  */
 public class ShowLoadProfileCommand extends ShowCommand {
-    private String loadIdPath;
+    public static final ShowResultSetMetaData META_DATA_QUERY_IDS;
+
+    static {
+        ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
+        for (String key : SummaryProfile.SUMMARY_CAPTIONS) {
+            if (key.equals(SummaryProfile.DISTRIBUTED_PLAN)) {
+                continue;
+            }
+            builder.addColumn(new Column(key, ScalarType.createStringType()));
+        }
+        META_DATA_QUERY_IDS = builder.build();
+    }
+
+    private long limit = 20;
 
     /**
      * constructor
      */
-    public ShowLoadProfileCommand(String path) {
+    public ShowLoadProfileCommand(String path, long limit) {
         super(PlanType.SHOW_LOAD_PROFILE_COMMAND);
-        this.loadIdPath = path;
+        this.limit = limit;
     }
 
     @Override
     public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
-        String selfHost = Env.getCurrentEnv().getSelfNode().getHost();
-        int httpPort = Config.http_port;
-        String terminalMsg = String.format(
-                "try visit http://%s:%d/QueryProfile/%s, show query/load profile syntax is a deprecated feature",
-                selfHost, httpPort, this.loadIdPath);
-        throw new UserException(terminalMsg);
+        List<List<String>> rows = ProfileManager.getInstance().getProfileMetaWithType(ProfileType.LOAD, this.limit);
+        return new ShowResultSet(META_DATA_QUERY_IDS, rows);
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowQueryProfileCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowQueryProfileCommand.java
index 1fe63619d0125e5..7c1732c33015b99 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowQueryProfileCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowQueryProfileCommand.java
@@ -17,39 +17,52 @@
 
 package org.apache.doris.nereids.trees.plans.commands;
 
-import org.apache.doris.catalog.Env;
-import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.Config;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.profile.ProfileManager;
+import org.apache.doris.common.profile.ProfileManager.ProfileType;
+import org.apache.doris.common.profile.SummaryProfile;
 import org.apache.doris.nereids.trees.plans.PlanType;
 import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.ShowResultSetMetaData;
 import org.apache.doris.qe.StmtExecutor;
 
+import java.util.List;
+
 /**
  * Represents the command for SHOW COLLATION
  */
 public class ShowQueryProfileCommand extends ShowCommand {
-    String queryIdPath;
+    public static final ShowResultSetMetaData META_DATA_QUERY_IDS;
+
+    static {
+        ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
+        for (String key : SummaryProfile.SUMMARY_CAPTIONS) {
+            if (key.equals(SummaryProfile.DISTRIBUTED_PLAN)) {
+                continue;
+            }
+            builder.addColumn(new Column(key, ScalarType.createStringType()));
+        }
+        META_DATA_QUERY_IDS = builder.build();
+    }
 
-    public ShowQueryProfileCommand(String queryIdPath) {
+    long limit = 20;
+
+    public ShowQueryProfileCommand(String queryIdPath, long limit) {
         super(PlanType.SHOW_QUERY_PROFILE_COMMAND);
-        this.queryIdPath = queryIdPath;
+        this.limit = limit;
     }
 
     @Override
     public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
-        String selfHost = Env.getCurrentEnv().getSelfNode().getHost();
-        int httpPort = Config.http_port;
-        String terminalMsg = String.format(
-                "try visit http://%s:%d/QueryProfile, show query/load profile syntax is a deprecated feature",
-                selfHost, httpPort);
-        throw new AnalysisException(terminalMsg);
+        List<List<String>> rows = ProfileManager.getInstance().getProfileMetaWithType(ProfileType.QUERY, this.limit);
+        return new ShowResultSet(META_DATA_QUERY_IDS, rows);
     }
 
     @Override
     public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
         return visitor.visitShowQueryProfileCommand(this, context);
     }
-
 }
diff --git a/regression-test/suites/query_profile/show_profile.groovy b/regression-test/suites/query_profile/show_profile.groovy
new file mode 100644
index 000000000000000..e0762f531e5d01a
--- /dev/null
+++ b/regression-test/suites/query_profile/show_profile.groovy
@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite('show_profile') {
+    sql "set enable_profile=true;"
+    sql "drop table if exists show_profile"
+    sql "create table show_profile (id int, name varchar(32)) distributed by hash(id) buckets 1 properties('replication_num'='1');"
+    sql "insert into show_profile values(1, 'a'), (2, 'b'), (3, 'c');"
+
+    for (int i = 0; i < 10; i++) {
+        sql "select * from show_profile;"
+    }
+
+    boolean executedFailed = false
+    try {
+        sql "show query profile;"
+        sql "show load profile;"
+    } catch (Exception e) {
+        logger.error("show query profile failed ", e)
+        executedFailed = true
+    }
+
+    assertEquals(false, executedFailed)
+
+    try {
+        sql """show query profile "/";"""
+        sql """show load profile "/";"""
+        executedFailed = false
+    } catch (Exception e) {
+        logger.error("show profile failed: {}", e)
+        executedFailed = true
+    }
+
+    assertEquals(false, executedFailed)
+
+    try {
+        sql """show query profile "/" limit 10;"""
+        sql """show load profile "/" limit 10;"""
+        executedFailed = false
+    } catch (Exception e) {
+        logger.error("show profile failed: {}", e)
+        executedFailed = true
+    }
+
+    assertEquals(false, executedFailed)
+}
\ No newline at end of file

From 6ab0713592b5ba80e5b15fdda0855ecc44f84418 Mon Sep 17 00:00:00 2001
From: shuke <shuke@selectdb.com>
Date: Thu, 9 Jan 2025 22:23:21 +0800
Subject: [PATCH 087/140] [regression-test](fix) fix
 test_default_current_timestamp.groovy (#46700)

### What problem does this PR solve?

Issue Number: close #xxx

Related PR: #xxx

Problem Summary:
This case assumes that the execution will not cross 24:00 during
runtime. Therefore, if the execution time is greater than 23:59:30, it
will wait until 24:00 to proceed.
---
 .../date/test_default_current_timestamp.groovy     | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/regression-test/suites/datatype_p0/date/test_default_current_timestamp.groovy b/regression-test/suites/datatype_p0/date/test_default_current_timestamp.groovy
index 21e61a1ef25242a..666aa189c68674b 100644
--- a/regression-test/suites/datatype_p0/date/test_default_current_timestamp.groovy
+++ b/regression-test/suites/datatype_p0/date/test_default_current_timestamp.groovy
@@ -16,7 +16,21 @@
 // specific language governing permissions and limitations
 // under the License.
 
+import java.time.LocalTime
+import java.time.format.DateTimeFormatter
+import java.time.Duration
+
 suite("test_default_current_timestamp") {
+    // This case assumes that the execution will not cross 24:00 during runtime. 
+    // Therefore, if the execution time is greater than 23:59:30, it will wait until 24:00 to proceed.
+    def t = sql "select now()"
+    LocalTime currentTime = t[0][0].toLocalTime()
+    if (currentTime.isAfter(LocalTime.of(23, 59, 30))) {
+        def s = Duration.between(currentTime, LocalTime.of(23, 59, 59)).seconds + 2
+        logger.info("sleep ${s} seconds to 24:00")
+        Thread.sleep(s * 1000)
+    } 
+
     def tbName = "test_default_current_timestamp"
     sql "DROP TABLE IF EXISTS ${tbName}"
     sql """

From 195319f4fe1cead18c1bc9804be294f125d9947e Mon Sep 17 00:00:00 2001
From: shuke <shuke@selectdb.com>
Date: Thu, 9 Jan 2025 22:23:41 +0800
Subject: [PATCH 088/140] [regression-test](fix) remove mow 2pc test cases
 (#46609)

---
 .../load_p0/stream_load/test_stream_load_properties.groovy    | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/regression-test/suites/load_p0/stream_load/test_stream_load_properties.groovy b/regression-test/suites/load_p0/stream_load/test_stream_load_properties.groovy
index 852ef2b8572380d..1ef25943681977f 100644
--- a/regression-test/suites/load_p0/stream_load/test_stream_load_properties.groovy
+++ b/regression-test/suites/load_p0/stream_load/test_stream_load_properties.groovy
@@ -549,6 +549,10 @@ suite("test_stream_load_properties", "p0") {
             sql new File("""${context.file.parent}/ddl/${tableName}_drop.sql""").text
             sql new File("""${context.file.parent}/ddl/${tableName}_create.sql""").text
 
+            if (isCloudMode()) {
+                continue
+            }
+
             String txnId
             def tableName1 =  "stream_load_" + tableName
             // Invalid txn_id string with letters

From e10209151490629d2507c1702c424d642fe9b56a Mon Sep 17 00:00:00 2001
From: Uniqueyou <wangyixuan@selectdb.com>
Date: Thu, 9 Jan 2025 23:46:35 +0800
Subject: [PATCH 089/140] [fix](ccr) Wrong tosql for alt tbl enable feature
 (#46620)

---
 .../plans/commands/AlterTableCommand.java     |  7 ++-
 .../plans/commands/AlterTableCommandTest.java | 53 +++++++++++++++++++
 2 files changed, 58 insertions(+), 2 deletions(-)
 create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java
index c17013b40879daa..675d2a11687a545 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java
@@ -58,6 +58,7 @@
 import org.apache.doris.qe.StmtExecutor;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -67,11 +68,13 @@
 public class AlterTableCommand extends Command implements ForwardWithSync {
     private TableNameInfo tbl;
     private List<AlterTableOp> ops;
+    private List<AlterTableOp> originOps;
 
     public AlterTableCommand(TableNameInfo tbl, List<AlterTableOp> ops) {
         super(PlanType.ALTER_TABLE_COMMAND);
         this.tbl = tbl;
         this.ops = ops;
+        this.originOps = ops;
     }
 
     public TableNameInfo getTbl() {
@@ -158,7 +161,7 @@ private void rewriteAlterOpForOlapTable(ConnectContext ctx, OlapTable table) thr
                 // analyse sequence column
                 Type sequenceColType = null;
                 if (alterFeature == EnableFeatureOp.Features.SEQUENCE_LOAD) {
-                    Map<String, String> propertyMap = alterClause.getProperties();
+                    Map<String, String> propertyMap = new HashMap<>(alterClause.getProperties());
                     try {
                         sequenceColType = PropertyAnalyzer.analyzeSequenceType(propertyMap, table.getKeysType());
                         if (sequenceColType == null) {
@@ -253,7 +256,7 @@ public String toSql() {
         StringBuilder sb = new StringBuilder();
         sb.append("ALTER TABLE ").append(tbl.toSql()).append(" ");
         int idx = 0;
-        for (AlterTableOp op : ops) {
+        for (AlterTableOp op : originOps) {
             if (idx != 0) {
                 sb.append(", \n");
             }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java
new file mode 100644
index 000000000000000..8f199da39ff051a
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java
@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.nereids.trees.plans.commands.info.AlterTableOp;
+import org.apache.doris.nereids.trees.plans.commands.info.EnableFeatureOp;
+import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class AlterTableCommandTest {
+    @Test
+    void testEnableFeatureOp() {
+        List<AlterTableOp> ops = new ArrayList<>();
+        ops.add(new EnableFeatureOp("BATCH_DELETE"));
+        AlterTableCommand alterTableCommand = new AlterTableCommand(new TableNameInfo("db", "test"), ops);
+        Assertions.assertEquals(alterTableCommand.toSql(), "ALTER TABLE `db`.`test` ENABLE FEATURE \"BATCH_DELETE\"");
+
+        ops.clear();
+        ops.add(new EnableFeatureOp("UPDATE_FLEXIBLE_COLUMNS"));
+        alterTableCommand = new AlterTableCommand(new TableNameInfo("db", "test"), ops);
+        Assertions.assertEquals(alterTableCommand.toSql(), "ALTER TABLE `db`.`test` ENABLE FEATURE \"UPDATE_FLEXIBLE_COLUMNS\"");
+
+        ops.clear();
+        Map<String, String> properties = new HashMap<>();
+        properties.put("function_column.sequence_type", "int");
+        ops.add(new EnableFeatureOp("SEQUENCE_LOAD", properties));
+        alterTableCommand = new AlterTableCommand(new TableNameInfo("db", "test"), ops);
+        Assertions.assertEquals(alterTableCommand.toSql(), "ALTER TABLE `db`.`test` ENABLE FEATURE \"SEQUENCE_LOAD\" WITH PROPERTIES (\"function_column.sequence_type\" = \"int\")");
+    }
+}
+

From 5683f65275b69bb3f5d04b7ea94336497aea253c Mon Sep 17 00:00:00 2001
From: meiyi <meiyi@selectdb.com>
Date: Thu, 9 Jan 2025 23:46:59 +0800
Subject: [PATCH 090/140] [fix](case) fix some cluster key case (#46634)

---
 .../test_compact_multi_segments.groovy                    | 4 +++-
 .../suites/unique_with_mow_c_p0/test_compact_seg.groovy   | 8 +++++++-
 .../unique_with_mow_c_p0/test_compact_with_seq.groovy     | 2 ++
 .../unique_with_mow_c_p0/test_compact_with_seq2.groovy    | 2 ++
 .../test_schema_change_add_key_column.groovy              | 2 ++
 .../test_schema_change_and_compaction.groovy              | 4 +++-
 6 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/regression-test/suites/unique_with_mow_c_p0/test_compact_multi_segments.groovy b/regression-test/suites/unique_with_mow_c_p0/test_compact_multi_segments.groovy
index d985729755f9d9b..3ef2d309758b676 100644
--- a/regression-test/suites/unique_with_mow_c_p0/test_compact_multi_segments.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/test_compact_multi_segments.groovy
@@ -16,12 +16,14 @@
 // under the License.
 
 suite("test_compact_multi_segments", "nonConcurrent") {
+    GetDebugPoint().clearDebugPointsForAllFEs()
+    GetDebugPoint().clearDebugPointsForAllBEs()
     def tableName = "test_compact_multi_segments"
 
     def getTabletStatus = { rowsetNum, lastRowsetSegmentNum ->
         def tablets = sql_return_maparray """ show tablets from ${tableName}; """
         logger.info("tablets: ${tablets}")
-        assertEquals(1, tablets.size())
+        assertTrue(tablets.size() >= 1)
         String compactionUrl = ""
         for (Map<String, String> tablet : tablets) {
             compactionUrl = tablet["CompactionStatus"]
diff --git a/regression-test/suites/unique_with_mow_c_p0/test_compact_seg.groovy b/regression-test/suites/unique_with_mow_c_p0/test_compact_seg.groovy
index 230653bcf3bd578..421e29564342a0e 100644
--- a/regression-test/suites/unique_with_mow_c_p0/test_compact_seg.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/test_compact_seg.groovy
@@ -16,12 +16,14 @@
 // under the License.
 
 suite("test_compact_seg", "nonConcurrent") {
+    GetDebugPoint().clearDebugPointsForAllFEs()
+    GetDebugPoint().clearDebugPointsForAllBEs()
     def tableName = "test_compact_seg"
 
     def getTabletStatus = { rowsetNum, lastRowsetSegmentNum ->
         def tablets = sql_return_maparray """ show tablets from ${tableName}; """
         logger.info("tablets: ${tablets}")
-        assertEquals(1, tablets.size())
+        assertTrue(tablets.size() >= 1)
         def tablet = tablets[0]
         String compactionUrl = tablet["CompactionStatus"]
         def retry = 15
@@ -99,6 +101,7 @@ suite("test_compact_seg", "nonConcurrent") {
             }
         }
         // check generate 3 segments
+        sql """ select * from ${tableName} limit 1; """
         getTabletStatus(2, 3)
 
         streamLoad {
@@ -118,6 +121,7 @@ suite("test_compact_seg", "nonConcurrent") {
             }
         }
         // check generate 2 segments(6 -> 2)
+        sql """ select * from ${tableName} limit 1; """
         getTabletStatus(3, 2)
 
         streamLoad {
@@ -137,6 +141,7 @@ suite("test_compact_seg", "nonConcurrent") {
             }
         }
         // check generate 2 segments(6 -> 2)
+        sql """ select * from ${tableName} limit 1; """
         getTabletStatus(4, 2)
 
         streamLoad {
@@ -156,6 +161,7 @@ suite("test_compact_seg", "nonConcurrent") {
             }
         }
         // check generate 2 segments(6 -> 2)
+        sql """ select * from ${tableName} limit 1; """
         getTabletStatus(5, 2)
 
         def rowCount1 = sql """ select count() from ${tableName}; """
diff --git a/regression-test/suites/unique_with_mow_c_p0/test_compact_with_seq.groovy b/regression-test/suites/unique_with_mow_c_p0/test_compact_with_seq.groovy
index bca3f1a1c09aff0..b9e08b21d7b8c1f 100644
--- a/regression-test/suites/unique_with_mow_c_p0/test_compact_with_seq.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/test_compact_with_seq.groovy
@@ -17,6 +17,8 @@
 
 // do compaction during commit rowset
 suite("test_compact_with_seq", "nonConcurrent") {
+    GetDebugPoint().clearDebugPointsForAllFEs()
+    GetDebugPoint().clearDebugPointsForAllBEs()
     def tableName = "test_compact_with_seq"
 
     def enable_publish_spin_wait = {
diff --git a/regression-test/suites/unique_with_mow_c_p0/test_compact_with_seq2.groovy b/regression-test/suites/unique_with_mow_c_p0/test_compact_with_seq2.groovy
index bef461bd464fc24..df50f5f07ea77fb 100644
--- a/regression-test/suites/unique_with_mow_c_p0/test_compact_with_seq2.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/test_compact_with_seq2.groovy
@@ -17,6 +17,8 @@
 
 // do compaction during publish
 suite("test_compact_with_seq2", "nonConcurrent") {
+    GetDebugPoint().clearDebugPointsForAllFEs()
+    GetDebugPoint().clearDebugPointsForAllBEs()
     def tableName = "test_compact_with_seq2"
     def dbId = getDbId()
 
diff --git a/regression-test/suites/unique_with_mow_c_p0/test_schema_change_add_key_column.groovy b/regression-test/suites/unique_with_mow_c_p0/test_schema_change_add_key_column.groovy
index 8f0a6d007b1425f..ba1a88e52461938 100644
--- a/regression-test/suites/unique_with_mow_c_p0/test_schema_change_add_key_column.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/test_schema_change_add_key_column.groovy
@@ -19,6 +19,8 @@ import org.awaitility.Awaitility
 import static java.util.concurrent.TimeUnit.SECONDS
 
 suite("test_schema_change_add_key_column", "nonConcurrent") {
+    GetDebugPoint().clearDebugPointsForAllFEs()
+    GetDebugPoint().clearDebugPointsForAllBEs()
     def tableName = "test_schema_change_add_key_column"
 
     def getAlterTableState = {
diff --git a/regression-test/suites/unique_with_mow_c_p0/test_schema_change_and_compaction.groovy b/regression-test/suites/unique_with_mow_c_p0/test_schema_change_and_compaction.groovy
index dfb7facf5ee4cbb..9190a6c114c6416 100644
--- a/regression-test/suites/unique_with_mow_c_p0/test_schema_change_and_compaction.groovy
+++ b/regression-test/suites/unique_with_mow_c_p0/test_schema_change_and_compaction.groovy
@@ -22,6 +22,8 @@
 import org.codehaus.groovy.runtime.IOGroovyMethods
 
 suite("test_schema_change_and_compaction", "nonConcurrent") {
+    GetDebugPoint().clearDebugPointsForAllFEs()
+    GetDebugPoint().clearDebugPointsForAllBEs()
     def tableName = "test_schema_change_and_compaction"
 
     def getAlterTableState = { job_state ->
@@ -78,7 +80,7 @@ suite("test_schema_change_and_compaction", "nonConcurrent") {
 
     def tablets = sql_return_maparray """ show tablets from ${tableName}; """
     logger.info("tablets: ${tablets}")
-    assertEquals(2, tablets.size())
+    assertTrue(tablets.size() >= 2)
     String alterTabletId = ""
     String alterTabletBackendId = ""
     String alterTabletCompactionUrl = ""

From 5a88bbaca0fcf9841db2200a875a0c256823a282 Mon Sep 17 00:00:00 2001
From: bobhan1 <baohan@selectdb.com>
Date: Thu, 9 Jan 2025 23:48:12 +0800
Subject: [PATCH 091/140] [Fix](regression) Fix define global var in regression
 case (#46531)

---
 .../fault_injection_p0/test_delete_from_timeout.groovy    | 2 +-
 .../test_full_compaciton_run_status.groovy                | 2 +-
 .../test_full_compaction_with_ordered_data.groovy         | 8 ++++----
 ...partial_update_rowset_not_found_fault_injection.groovy | 4 ++--
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/regression-test/suites/fault_injection_p0/test_delete_from_timeout.groovy b/regression-test/suites/fault_injection_p0/test_delete_from_timeout.groovy
index 8598d791e01106e..b15309891da0674 100644
--- a/regression-test/suites/fault_injection_p0/test_delete_from_timeout.groovy
+++ b/regression-test/suites/fault_injection_p0/test_delete_from_timeout.groovy
@@ -48,7 +48,7 @@ suite("test_delete_from_timeout","nonConcurrent") {
 
         GetDebugPoint().enableDebugPointForAllBEs("PushHandler::_do_streaming_ingestion.try_lock_fail")
 
-        t1 = Thread.start {
+        def t1 = Thread.start {
             sleep(15000)
             GetDebugPoint().disableDebugPointForAllBEs("PushHandler::_do_streaming_ingestion.try_lock_fail")
         }
diff --git a/regression-test/suites/fault_injection_p0/test_full_compaciton_run_status.groovy b/regression-test/suites/fault_injection_p0/test_full_compaciton_run_status.groovy
index 29f80528a80b41e..dffcfb293f1298d 100644
--- a/regression-test/suites/fault_injection_p0/test_full_compaciton_run_status.groovy
+++ b/regression-test/suites/fault_injection_p0/test_full_compaciton_run_status.groovy
@@ -65,7 +65,7 @@ suite("test_full_compaction_run_status","nonConcurrent") {
             String tablet_id = tablet.TabletId
             backend_id = tablet.BackendId
 
-            times = 1
+            def times = 1
             do{
                 (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
                 logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
diff --git a/regression-test/suites/fault_injection_p0/test_full_compaction_with_ordered_data.groovy b/regression-test/suites/fault_injection_p0/test_full_compaction_with_ordered_data.groovy
index c060888690c0772..6d741ddfedd7131 100644
--- a/regression-test/suites/fault_injection_p0/test_full_compaction_with_ordered_data.groovy
+++ b/regression-test/suites/fault_injection_p0/test_full_compaction_with_ordered_data.groovy
@@ -60,7 +60,7 @@ suite("test_full_compaction_with_ordered_data","nonConcurrent") {
         int rowsetCount = 0
         for (def tablet in tablets) {
             String tablet_id = tablet.TabletId
-            (code, out, err) = curl("GET", tablet.CompactionStatus)
+            def (code, out, err) = curl("GET", tablet.CompactionStatus)
             logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
             assertEquals(code, 0)
             def tabletJson = parseJson(out.trim())
@@ -76,7 +76,7 @@ suite("test_full_compaction_with_ordered_data","nonConcurrent") {
         rowsetCount = 0
         for (def tablet in tablets) {
             String tablet_id = tablet.TabletId
-            (code, out, err) = curl("GET", tablet.CompactionStatus)
+            def (code, out, err) = curl("GET", tablet.CompactionStatus)
             logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
             assertEquals(code, 0)
             def tabletJson = parseJson(out.trim())
@@ -116,7 +116,7 @@ suite("test_full_compaction_with_ordered_data","nonConcurrent") {
         int rowsetCount = 0
         for (def tablet in tablets) {
             String tablet_id = tablet.TabletId
-            (code, out, err) = curl("GET", tablet.CompactionStatus)
+            def (code, out, err) = curl("GET", tablet.CompactionStatus)
             logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
             assertEquals(code, 0)
             def tabletJson = parseJson(out.trim())
@@ -132,7 +132,7 @@ suite("test_full_compaction_with_ordered_data","nonConcurrent") {
         rowsetCount = 0
         for (def tablet in tablets) {
             String tablet_id = tablet.TabletId
-            (code, out, err) = curl("GET", tablet.CompactionStatus)
+            def (code, out, err) = curl("GET", tablet.CompactionStatus)
             logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
             assertEquals(code, 0)
             def tabletJson = parseJson(out.trim())
diff --git a/regression-test/suites/fault_injection_p2/test_partial_update_rowset_not_found_fault_injection.groovy b/regression-test/suites/fault_injection_p2/test_partial_update_rowset_not_found_fault_injection.groovy
index befad64da0a3bfb..39344550526aaeb 100644
--- a/regression-test/suites/fault_injection_p2/test_partial_update_rowset_not_found_fault_injection.groovy
+++ b/regression-test/suites/fault_injection_p2/test_partial_update_rowset_not_found_fault_injection.groovy
@@ -94,8 +94,8 @@ suite("test_partial_update_rowset_not_found_fault_injection", "p2,nonConcurrent"
         def tablets = sql_return_maparray """ show tablets from ${testTable}; """
         for (def tablet in tablets) {
             String tablet_id = tablet.TabletId
-            backend_id = tablet.BackendId
-            (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+            def backend_id = tablet.BackendId
+            def (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
             logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
             assertEquals(code, 0)
         }

From 827e80769041bfde19db2311959f17e6c8f658f0 Mon Sep 17 00:00:00 2001
From: huanghaibin <huanghaibin@selectdb.com>
Date: Thu, 9 Jan 2025 23:53:30 +0800
Subject: [PATCH 092/140] [improve](mow) add observability on commit and
 publish cost time (#46487)

Recording the time on commit and publish phase is helpful to know
whether mow cost too much time on this phase.
---
 be/src/cloud/cloud_stream_load_executor.cpp   |  7 ++++
 be/src/http/action/stream_load.cpp            |  4 ++
 .../CloudGlobalTransactionMgr.java            | 40 ++++++++++++-------
 .../org/apache/doris/metric/MetricRepo.java   |  5 +++
 4 files changed, 42 insertions(+), 14 deletions(-)

diff --git a/be/src/cloud/cloud_stream_load_executor.cpp b/be/src/cloud/cloud_stream_load_executor.cpp
index 46ceca851e2b4df..4952b29df0da0b1 100644
--- a/be/src/cloud/cloud_stream_load_executor.cpp
+++ b/be/src/cloud/cloud_stream_load_executor.cpp
@@ -17,6 +17,8 @@
 
 #include "cloud/cloud_stream_load_executor.h"
 
+#include <bvar/bvar.h>
+
 #include "cloud/cloud_meta_mgr.h"
 #include "cloud/cloud_storage_engine.h"
 #include "cloud/config.h"
@@ -27,6 +29,10 @@
 
 namespace doris {
 
+bvar::Adder<uint64_t> stream_load_commit_retry_counter;
+bvar::Window<bvar::Adder<uint64_t>> stream_load_commit_retry_counter_minute(
+        &stream_load_commit_retry_counter, 60);
+
 enum class TxnOpParamType : int {
     ILLEGAL,
     WITH_TXN_ID,
@@ -114,6 +120,7 @@ Status CloudStreamLoadExecutor::commit_txn(StreamLoadContext* ctx) {
                     .tag("retry_times", retry_times)
                     .error(st);
             retry_times++;
+            stream_load_commit_retry_counter << 1;
         }
         return st;
     }
diff --git a/be/src/http/action/stream_load.cpp b/be/src/http/action/stream_load.cpp
index e8db5cb542fb4bf..9f6d1d56aad45eb 100644
--- a/be/src/http/action/stream_load.cpp
+++ b/be/src/http/action/stream_load.cpp
@@ -77,6 +77,8 @@ DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(streaming_load_duration_ms, MetricUnit::MIL
 DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(streaming_load_current_processing, MetricUnit::REQUESTS);
 
 bvar::LatencyRecorder g_stream_load_receive_data_latency_ms("stream_load_receive_data_latency_ms");
+bvar::LatencyRecorder g_stream_load_commit_and_publish_latency_ms("stream_load",
+                                                                  "commit_and_publish_ms");
 
 static constexpr size_t MIN_CHUNK_SIZE = 64 * 1024;
 static const string CHUNK = "chunked";
@@ -185,6 +187,8 @@ Status StreamLoadAction::_handle(std::shared_ptr<StreamLoadContext> ctx) {
         int64_t commit_and_publish_start_time = MonotonicNanos();
         RETURN_IF_ERROR(_exec_env->stream_load_executor()->commit_txn(ctx.get()));
         ctx->commit_and_publish_txn_cost_nanos = MonotonicNanos() - commit_and_publish_start_time;
+        g_stream_load_commit_and_publish_latency_ms
+                << ctx->commit_and_publish_txn_cost_nanos / 1000000;
     }
     return Status::OK();
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
index 2ff882bcbfc9d7e..0084e677d21da97 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
@@ -1047,26 +1047,38 @@ private void debugCalcDeleteBitmapRandomTimeout() throws UserException {
     public boolean commitAndPublishTransaction(DatabaseIf db, List<Table> tableList, long transactionId,
                                                List<TabletCommitInfo> tabletCommitInfos, long timeoutMillis)
             throws UserException {
+        StopWatch stopWatch = new StopWatch();
+        stopWatch.start();
         int retryTimes = 0;
         boolean res = false;
-        while (true) {
-            try {
-                res = commitAndPublishTransaction(db, tableList, transactionId, tabletCommitInfos, timeoutMillis, null);
-                break;
-            } catch (UserException e) {
-                LOG.warn("failed to commit txn, txnId={},retryTimes={},exception={}",
-                        transactionId, retryTimes, e);
-                // only mow table will catch DELETE_BITMAP_LOCK_ERR and need to retry
-                if (e.getErrorCode() == InternalErrorCode.DELETE_BITMAP_LOCK_ERR) {
-                    retryTimes++;
-                    if (retryTimes >= Config.mow_calculate_delete_bitmap_retry_times) {
-                        // should throw exception after running out of retry times
+        try {
+            while (true) {
+                try {
+                    res = commitAndPublishTransaction(db, tableList, transactionId, tabletCommitInfos, timeoutMillis,
+                            null);
+                    break;
+                } catch (UserException e) {
+                    LOG.warn("failed to commit txn, txnId={},retryTimes={},exception={}",
+                            transactionId, retryTimes, e);
+                    // only mow table will catch DELETE_BITMAP_LOCK_ERR and need to retry
+                    if (e.getErrorCode() == InternalErrorCode.DELETE_BITMAP_LOCK_ERR) {
+                        retryTimes++;
+                        if (retryTimes >= Config.mow_calculate_delete_bitmap_retry_times) {
+                            // should throw exception after running out of retry times
+                            throw e;
+                        }
+                    } else {
                         throw e;
                     }
-                } else {
-                    throw e;
                 }
             }
+        } finally {
+            stopWatch.stop();
+            long commitAndPublishTime = stopWatch.getTime();
+            LOG.info("commitAndPublishTransaction txn {} cost {} ms", transactionId, commitAndPublishTime);
+            if (MetricRepo.isInit) {
+                MetricRepo.HISTO_COMMIT_AND_PUBLISH_LATENCY.update(commitAndPublishTime);
+            }
         }
         return res;
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
index cda73ecbe6221d1..2f87515509a764e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
@@ -147,6 +147,8 @@ public final class MetricRepo {
     public static GaugeMetricImpl<Double> GAUGE_QUERY_SLOW_RATE;
     public static GaugeMetricImpl<Long> GAUGE_MAX_TABLET_COMPACTION_SCORE;
 
+    public static Histogram HISTO_COMMIT_AND_PUBLISH_LATENCY;
+
     private static Map<Pair<EtlJobType, JobState>, Long> loadJobNum = Maps.newHashMap();
 
     private static ScheduledThreadPoolExecutor metricTimer = ThreadPoolManager.newDaemonScheduledThreadPool(1,
@@ -555,6 +557,9 @@ public Long getValue() {
         HISTO_HTTP_COPY_INTO_QUERY_LATENCY = METRIC_REGISTER.histogram(
             MetricRegistry.name("http_copy_into_query", "latency", "ms"));
 
+        HISTO_COMMIT_AND_PUBLISH_LATENCY = METRIC_REGISTER.histogram(
+                MetricRegistry.name("txn_commit_and_publish", "latency", "ms"));
+
         // init system metrics
         initSystemMetrics();
         CloudMetrics.init();

From 4fe222cdd4d4b2e561b2eac573b52a1052268971 Mon Sep 17 00:00:00 2001
From: Calvin Kirs <guoqiang@selectdb.com>
Date: Fri, 10 Jan 2025 09:50:01 +0800
Subject: [PATCH 093/140] [Fix](insert-job)Fix Issue with Missing
 StatementContext in StmtExecutor, Leading to Incorrect SQL_BLOCK Handling
 (#46715)

### What problem does this PR solve?

During the task construction process of the Insert Job, the StmtExecutor
was not provided with a corresponding StatementContext, which causes a
default StatementContext to be generated. This can lead to issues where
certain properties related to StatementContext, such as the globally set
SQL_BLOCK, do not function correctly.

**How to reproduce**
```
mysql> create job test on SCHEDULE every 2 second do INSERT INTO image values ('2023-10-13',1,2);
Query OK, 0 rows affected (0.07 sec)
mysql> CREATE SQL_BLOCK_RULE test_sql_block
    -> PROPERTIES(
    -> "sql"="select \\* from order_analysis",
    -> "global"="true",
    -> "enable"="true"
    -> );


 2025-01-09 18:38:03 |             | {"Unfinished backends":{},"ScannedRows":0,"TaskNumber":0,"LoadBytes":0,"All backends":{},"FileNumber":0,"FileSize":0} | root |
| 7377995488171 | 7170662367696 | test    | 7170662367696_7377995488171 | SUCCESS |                                                                                                             | 2025-01-09 18:38:05 | 2025-01-09 18:38:05 | 2025-01-09 18:38:05 |             | {"Unfinished backends":{},"ScannedRows":0,"TaskNumber":0,"LoadBytes":0,"All backends":{},"FileNumber":0,"FileSize":0} | root |
| 7379811123912 | 7170662367696 | test    | 7170662367696_7379811123912 | FAILED  | java.lang.NullPointerException: Cannot invoke "java.lang.CharSequence.length()" because "this.text" is null | 2025-01-09 18:38:07 | 2025-01-09 18:38:07 |                     |             |                                                                                                                       | root |
| 7382097155737 | 7170662367696 | test    | 7170662367696_7382097155737 | FAILED  | java.lang.NullPointerException: Cannot invoke "java.lang.CharSequence.length()" because "this.text" is null | 2025-01-09 18:38:09 | 2025-01-09 18:38:09 |                     |             |                                                                                                                       | root |
| 7383760717194 | 7170662367696 | test    | 7170662367696_7383760717194 | FAILED  | java.lang.NullPointerException: Cannot invoke "java.lang.CharSequence.length()" because "this.text" is null | 2025-01-09 18:38:11 | 2025-01-09 18:38:11 |                     |             |                                                                                                                       | root |
| 7385509219729 | 7170662367696 | test    | 7170662367696_7385509219729 | FAILED  | java.lang.NullPointerException: Cannot invoke "java.lang.CharSequence.length()" because "this.text" is null | 2025-01-09 18:38:13 | 2025-01-09 18:38:13 |                     |             |                                                                                                                       | root |
| 7388113399522 | 7170662367696 | test    | 7170662367696_7388113399522 | FAILED  | java.lang.NullPointerException: Cannot invoke "java.lang.CharSequence.length()" because "this.text" is null | 2025-01-09 18:38:15 | 2025-01-09 18:38:15 |                     |             |                                                                                                                       | root |
| 7389678516159 | 7170662367696 | test    | 7170662367696_7389678516159 | FAILED  | java.lang.NullPointerException: Cannot invoke "java.lang.CharSequence.length()" because "this.text" is null | 2025-01-09 18:38:17 | 2025-01-09 18:38:17 |                     |             |                                                                                                                       | root |
| 7391506550186 | 7170662367696 | test    | 7170662367696_7391506550186 | FAILED  | java.lang.NullPointerException: Cannot invoke "java.lang.CharSequence.length()" because "this.text" is null | 2025-01-09 18:38:19 | 2025-01-09 18:38:19 |                     |             |                                                                                                                       | root |
| 7393442362259 | 7170662367696 | test    | 7170662367696_7393442362259 | FAILED  | java.lang.NullPointerException: Cannot invoke "java.lang.CharSequence.length()" because "this.text" is null | 2025-01-09 18:38:21 | 2025-01-09 18:38:21 |                     |             |                                                                                                                       | root |
+---------------+---------------+---------+-----------------------------+---------+-------------------------------------------------------------------------------------------------------------+---------------------+---------------------+---------------------+-------------+-----------------------------------------------------------------------------------------------------------------------+------+
62 rows in set (0.14 sec)
```
---
 .../doris/job/extensions/insert/InsertTask.java    |  6 +++++-
 .../suites/job_p0/test_base_insert_job.groovy      | 14 ++++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java
index 883c42653167a29..ff18f611c1ed8db 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java
@@ -29,6 +29,8 @@
 import org.apache.doris.load.FailMsg;
 import org.apache.doris.load.loadv2.LoadJob;
 import org.apache.doris.load.loadv2.LoadStatistic;
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.glue.LogicalPlanAdapter;
 import org.apache.doris.nereids.parser.NereidsParser;
 import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand;
 import org.apache.doris.qe.ConnectContext;
@@ -159,13 +161,15 @@ public void before() throws JobException {
             ctx.setDatabase(currentDb);
         }
         TUniqueId queryId = generateQueryId(UUID.randomUUID().toString());
-        stmtExecutor = new StmtExecutor(ctx, (String) null);
+        StatementContext statementContext = new StatementContext();
         ctx.setQueryId(queryId);
+        ctx.setStatementContext(statementContext);
         if (StringUtils.isNotEmpty(sql)) {
             NereidsParser parser = new NereidsParser();
             this.command = (InsertIntoTableCommand) parser.parseSingle(sql);
             this.command.setLabelName(Optional.of(getJobId() + LABEL_SPLITTER + getTaskId()));
             this.command.setJobId(getTaskId());
+            stmtExecutor = new StmtExecutor(ctx, new LogicalPlanAdapter(command, ctx.getStatementContext()));
         }
 
         super.before();
diff --git a/regression-test/suites/job_p0/test_base_insert_job.groovy b/regression-test/suites/job_p0/test_base_insert_job.groovy
index 1703b355c950192..042546d1e430b40 100644
--- a/regression-test/suites/job_p0/test_base_insert_job.groovy
+++ b/regression-test/suites/job_p0/test_base_insert_job.groovy
@@ -72,6 +72,17 @@ suite("test_base_insert_job") {
             "replication_allocation" = "tag.location.default: 1"
         );
         """
+    sql """
+       DROP SQL_BLOCK_RULE IF EXISTS test_base_insert_job_rule
+        """
+    sql """
+        CREATE SQL_BLOCK_RULE test_base_insert_job_rule
+            PROPERTIES(
+            "sql"="select \\\\* from test_base_insert_job_rule",
+            "global"="true",
+            "enable"="true"
+            );
+        """
     sql """
         insert into ${tableName} values
         ('2023-03-18', 1, 1)
@@ -242,6 +253,9 @@ suite("test_base_insert_job") {
         //resume tasks size should be greater than before pause
         afterResumeTasks.size() > tasks.size()
     })
+    sql """
+       DROP SQL_BLOCK_RULE IF EXISTS test_base_insert_job_rule
+        """
     // check resume job status
     def afterResumeJobStatus = sql """
         select status from jobs("type"="insert") where Name='${jobName}'

From 66dddc2d1b8e34eeecc0e234a572733d3a4e2a0d Mon Sep 17 00:00:00 2001
From: walter <maochuan@selectdb.com>
Date: Fri, 10 Jan 2025 10:02:24 +0800
Subject: [PATCH 094/140] [chore](persist) Save old index schema map in
 TableAddOrDropColumnsInfo (#46606)

---
 .../org/apache/doris/alter/SchemaChangeHandler.java   | 11 ++++++-----
 .../org/apache/doris/alter/SchemaChangeJobV2.java     |  4 ++--
 .../main/java/org/apache/doris/catalog/OlapTable.java |  9 +++++++++
 .../doris/persist/TableAddOrDropColumnsInfo.java      |  8 +++++++-
 .../doris/persist/TableAddOrDropColumnsInfoTest.java  |  5 ++++-
 5 files changed, 28 insertions(+), 9 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index 72d2419177c1892..af2a0a7a9bd90e1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -2031,16 +2031,16 @@ public int getAsInt() {
                     }
                 } else if (alterClause instanceof AddColumnsClause) {
                     // add columns
-                    boolean clauseCanLigthSchemaChange = processAddColumns((AddColumnsClause) alterClause, olapTable,
+                    boolean clauseCanLightSchemaChange = processAddColumns((AddColumnsClause) alterClause, olapTable,
                             indexSchemaMap, false, colUniqueIdSupplierMap);
-                    if (!clauseCanLigthSchemaChange) {
+                    if (!clauseCanLightSchemaChange) {
                         lightSchemaChange = false;
                     }
                 } else if (alterClause instanceof DropColumnClause) {
                     // drop column and drop indexes on this column
-                    boolean clauseCanLigthSchemaChange = processDropColumn((DropColumnClause) alterClause, olapTable,
+                    boolean clauseCanLightSchemaChange = processDropColumn((DropColumnClause) alterClause, olapTable,
                             indexSchemaMap, newIndexes);
-                    if (!clauseCanLigthSchemaChange) {
+                    if (!clauseCanLightSchemaChange) {
                         lightSchemaChange = false;
                     }
                 } else if (alterClause instanceof ModifyColumnClause) {
@@ -2890,6 +2890,7 @@ public void modifyTableLightSchemaChange(String rawSql, Database db, OlapTable o
         }
 
         //update base index schema
+        Map<Long, List<Column>> oldIndexSchemaMap = olapTable.getCopiedIndexIdToSchema(true);
         try {
             updateBaseIndexSchema(olapTable, indexSchemaMap, indexes);
         } catch (Exception e) {
@@ -2935,7 +2936,7 @@ public void modifyTableLightSchemaChange(String rawSql, Database db, OlapTable o
         } else {
             if (!isReplay) {
                 TableAddOrDropColumnsInfo info = new TableAddOrDropColumnsInfo(rawSql, db.getId(), olapTable.getId(),
-                        indexSchemaMap, indexes, jobId);
+                        indexSchemaMap, oldIndexSchemaMap, indexes, jobId);
                 if (LOG.isDebugEnabled()) {
                     LOG.debug("logModifyTableAddOrDropColumns info:{}", info);
                 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
index bea703a29d75fa5..f031dee3635879b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
@@ -622,7 +622,7 @@ protected void runRunningJob() throws AlterCancelException {
                 Partition partition = tbl.getPartition(partitionId);
                 Preconditions.checkNotNull(partition, partitionId);
 
-                long visiableVersion = partition.getVisibleVersion();
+                long visibleVersion = partition.getVisibleVersion();
                 short expectReplicationNum = tbl.getPartitionInfo()
                         .getReplicaAllocation(partition.getId()).getTotalReplicaNum();
 
@@ -635,7 +635,7 @@ protected void runRunningJob() throws AlterCancelException {
                         int healthyReplicaNum = 0;
                         for (Replica replica : replicas) {
                             if (!replica.isBad() && replica.getLastFailedVersion() < 0
-                                    && replica.checkVersionCatchUp(visiableVersion, false)) {
+                                    && replica.checkVersionCatchUp(visibleVersion, false)) {
                                 healthyReplicaNum++;
                             }
                         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 9eba9499d637073..9df096fc4219b39 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -933,6 +933,15 @@ public Map<Long, List<Column>> getIndexIdToSchema(boolean full) {
         return result;
     }
 
+    // get schemas with a copied column list
+    public Map<Long, List<Column>> getCopiedIndexIdToSchema(boolean full) {
+        Map<Long, List<Column>> result = Maps.newHashMap();
+        for (Map.Entry<Long, MaterializedIndexMeta> entry : indexIdToMeta.entrySet()) {
+            result.put(entry.getKey(), new ArrayList<>(entry.getValue().getSchema(full)));
+        }
+        return result;
+    }
+
     public List<Column> getSchemaByIndexId(Long indexId) {
         return getSchemaByIndexId(indexId, Util.showHiddenColumns());
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/TableAddOrDropColumnsInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/TableAddOrDropColumnsInfo.java
index 9939608067462fd..14b7187fac671fd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/TableAddOrDropColumnsInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/TableAddOrDropColumnsInfo.java
@@ -42,6 +42,8 @@ public class TableAddOrDropColumnsInfo implements Writable {
     private long tableId;
     @SerializedName(value = "indexSchemaMap")
     private Map<Long, LinkedList<Column>> indexSchemaMap;
+    @SerializedName(value = "oldIndexSchemaMap")
+    private Map<Long, List<Column>> oldIndexSchemaMap;  // only used for ccr, not included in equals
     @SerializedName(value = "indexes")
     private List<Index> indexes;
     @SerializedName(value = "jobId")
@@ -50,11 +52,14 @@ public class TableAddOrDropColumnsInfo implements Writable {
     private String rawSql;
 
     public TableAddOrDropColumnsInfo(String rawSql, long dbId, long tableId,
-            Map<Long, LinkedList<Column>> indexSchemaMap, List<Index> indexes, long jobId) {
+            Map<Long, LinkedList<Column>> indexSchemaMap,
+            Map<Long, List<Column>> oldIndexSchemaMap,
+            List<Index> indexes, long jobId) {
         this.rawSql = rawSql;
         this.dbId = dbId;
         this.tableId = tableId;
         this.indexSchemaMap = indexSchemaMap;
+        this.oldIndexSchemaMap = oldIndexSchemaMap;
         this.indexes = indexes;
         this.jobId = jobId;
     }
@@ -111,6 +116,7 @@ public String toString() {
         sb.append(" dbId: ").append(dbId);
         sb.append(" tableId: ").append(tableId);
         sb.append(" indexSchemaMap: ").append(indexSchemaMap);
+        sb.append(" oldIndexSchemaMap: ").append(oldIndexSchemaMap);
         sb.append(" indexes: ").append(indexes);
         sb.append(" jobId: ").append(jobId);
         return sb.toString();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java
index be71998eac3f778..4df2759e9df92a7 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java
@@ -64,11 +64,14 @@ public void testSerialization() throws IOException {
         Map<Long, LinkedList<Column>> indexSchemaMap = new HashMap<>();
         indexSchemaMap.put(tableId, fullSchema);
 
+        Map<Long, List<Column>> oldIndexSchemaMap = new HashMap<>();
+        oldIndexSchemaMap.put(tableId, fullSchema);
+
         List<Index> indexes = Lists.newArrayList(
                 new Index(0, "index", Lists.newArrayList("testCol1"), IndexDef.IndexType.INVERTED, null, "xxxxxx", Lists.newArrayList(1)));
 
         TableAddOrDropColumnsInfo tableAddOrDropColumnsInfo1 = new TableAddOrDropColumnsInfo("", dbId, tableId,
-                indexSchemaMap, indexes, jobId);
+                indexSchemaMap, oldIndexSchemaMap, indexes, jobId);
 
         String c1Json = GsonUtils.GSON.toJson(tableAddOrDropColumnsInfo1);
         Text.writeString(out, c1Json);

From fb5e5553894c855f0ae0899410e9e32c4cf38801 Mon Sep 17 00:00:00 2001
From: "Mingyu Chen (Rayner)" <morningman@163.com>
Date: Fri, 10 Jan 2025 10:22:38 +0800
Subject: [PATCH 095/140] [fix](test) fix unstable jdbc ddl case (#46697)

### What problem does this PR solve?

When `use_meta_cache = false`, we should wait for database sync.
---
 .../test_nereids_refresh_catalog.out          | 39 -------------------
 .../jdbc/test_jdbc_catalog_ddl.groovy         | 21 ++++++++++
 .../test_nereids_refresh_catalog.groovy       | 10 ++---
 3 files changed, 26 insertions(+), 44 deletions(-)

diff --git a/regression-test/data/external_table_p0/nereids_commands/test_nereids_refresh_catalog.out b/regression-test/data/external_table_p0/nereids_commands/test_nereids_refresh_catalog.out
index 66ca9363d711dc1..a71ca04ae667789 100644
--- a/regression-test/data/external_table_p0/nereids_commands/test_nereids_refresh_catalog.out
+++ b/regression-test/data/external_table_p0/nereids_commands/test_nereids_refresh_catalog.out
@@ -1,13 +1,6 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !database --
-DORIS
-Doris
-doris
 doris_test
-information_schema
-init_db
-mysql
-show_test_do_not_modify
 
 -- !ex_tb0_where --
 111
@@ -20,25 +13,9 @@ show_test_do_not_modify
 115	abg
 
 -- !preceding_create_external_database --
-DORIS
-Doris
-doris
-doris_test
-information_schema
-init_db
-mysql
-show_test_do_not_modify
 
 -- !subsequent_create_external_database --
-DORIS
-Doris
-doris
-doris_test
-information_schema
-init_db
-mysql
 new_mysql_db
-show_test_do_not_modify
 
 -- !sql_show_tables --
 
@@ -76,23 +53,7 @@ new_mysql_table2	CREATE TABLE `new_mysql_table2` (\n  `id` int NULL,\n  `name` v
 new_mysql_table2	CREATE TABLE `new_mysql_table2` (\n  `id` int NULL,\n  `name` varchar(20) NULL,\n  `new_column` int NULL,\n  `new_column_1` int NULL,\n  `new_column_2` int NULL\n) ENGINE=JDBC_EXTERNAL_TABLE;
 
 -- !preceding_drop_external_database --
-DORIS
-Doris
-doris
-doris_test
-information_schema
-init_db
-mysql
 new_mysql_db
-show_test_do_not_modify
 
 -- !subsequent_drop_external_database --
-DORIS
-Doris
-doris
-doris_test
-information_schema
-init_db
-mysql
-show_test_do_not_modify
 
diff --git a/regression-test/suites/external_table_p0/jdbc/test_jdbc_catalog_ddl.groovy b/regression-test/suites/external_table_p0/jdbc/test_jdbc_catalog_ddl.groovy
index 3735fe5305396bc..1750f732baf7451 100644
--- a/regression-test/suites/external_table_p0/jdbc/test_jdbc_catalog_ddl.groovy
+++ b/regression-test/suites/external_table_p0/jdbc/test_jdbc_catalog_ddl.groovy
@@ -15,6 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
+import java.util.concurrent.TimeUnit;
+import org.awaitility.Awaitility;
+
 suite("test_jdbc_catalog_ddl", "p0,external,mysql,external_docker,external_docker_mysql") {
 
     String enabled = context.config.otherConfigs.get("enableJdbcTest")
@@ -23,6 +26,18 @@ suite("test_jdbc_catalog_ddl", "p0,external,mysql,external_docker,external_docke
     String bucket = getS3BucketName()
     String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-java-5.1.49.jar"
     String mysql_port = context.config.otherConfigs.get("mysql_57_port");
+
+    def wait_db_sync = { String ctl ->
+        Awaitility.await().atMost(10, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).until{
+            try {
+                def res = sql "show databases from ${ctl}"
+                return res.size() > 0;
+            } catch (Exception e) {
+                e.printStackTrace();
+                return false;
+            }
+        }
+    }
     // String driver_url = "mysql-connector-java-5.1.49.jar"
     if (enabled != null && enabled.equalsIgnoreCase("true")) {
         String catalog_name = "test_jdbc_catalog_ddl";
@@ -38,6 +53,11 @@ suite("test_jdbc_catalog_ddl", "p0,external,mysql,external_docker,external_docke
                 "driver_class" = "com.mysql.jdbc.Driver",
                 "use_meta_cache" = "${useMetaCache}"
             );"""
+
+            if (useMetaCache.equals("false")) {
+                wait_db_sync("${catalog_name}")
+            }
+
             def res = sql(""" show databases from ${catalog_name}; """).collect {x -> x[0] as String}
             println("show databases result " + res);
             def containedDb = ['mysql', 'doris_test', 'information_schema']
@@ -68,6 +88,7 @@ suite("test_jdbc_catalog_ddl", "p0,external,mysql,external_docker,external_docke
 
             if (useMetaCache.equals("false")) {
                 sql """refresh catalog ${catalog_name}"""
+                wait_db_sync("${catalog_name}")
             }
             sql "use ${catalog_name}.temp_database"
             qt_sql01 """select * from temp_table"""
diff --git a/regression-test/suites/external_table_p0/nereids_commands/test_nereids_refresh_catalog.groovy b/regression-test/suites/external_table_p0/nereids_commands/test_nereids_refresh_catalog.groovy
index 8fbaa486610109b..e5ffbed9c219da8 100644
--- a/regression-test/suites/external_table_p0/nereids_commands/test_nereids_refresh_catalog.groovy
+++ b/regression-test/suites/external_table_p0/nereids_commands/test_nereids_refresh_catalog.groovy
@@ -48,16 +48,16 @@ suite("test_nereids_refresh_catalog", "p0,external,mysql,external_docker,externa
         sql """CALL EXECUTE_STMT("${catalog_name}", "drop database if exists ${new_mysql_db}");"""
         sql """switch ${catalog_name}"""
 
-        qt_database """show databases;"""
+        qt_database """show databases like "%doris_test%";"""
         sql """ use ${ex_db_name}"""
 
         qt_ex_tb0_where """select id from ${ex_tb0} where id = 111;"""
         order_qt_ex_tb0  """ select id, name from ${ex_tb0} order by id; """
         // create database in mysql
         sql """CALL EXECUTE_STMT("${catalog_name}", "create database  ${new_mysql_db} ;");"""
-        qt_preceding_create_external_database """show databases;"""
+        qt_preceding_create_external_database """show databases like "%${new_mysql_db}%";"""
         checkNereidsExecute("refresh catalog ${catalog_name} ;")
-        qt_subsequent_create_external_database """show databases;"""
+        qt_subsequent_create_external_database """show databases like "%${new_mysql_db}%";"""
 
         checkNereidsExecute("refresh catalog ${catalog_name} properties ('invalid_cache'='true');")
         sql """use ${new_mysql_db}"""
@@ -94,10 +94,10 @@ suite("test_nereids_refresh_catalog", "p0,external,mysql,external_docker,externa
         qt_subsequent_refresh_table """show create table ${new_mysql_db}.${new_mysql_table2};"""
 
         sql """CALL EXECUTE_STMT("${catalog_name}", "drop database if exists ${new_mysql_db} ;");"""
-        qt_preceding_drop_external_database """show databases;"""
+        qt_preceding_drop_external_database """show databases like "%${new_mysql_db}%";"""
 
         checkNereidsExecute("refresh catalog ${catalog_name} properties ('invalid_cache'='true');")
-        qt_subsequent_drop_external_database """show databases;"""
+        qt_subsequent_drop_external_database """show databases like "%${new_mysql_db}%";"""
 
         sql """ drop catalog if exists ${catalog_name} ;"""
     }

From 7a6c12524b1df04bd7b419c8e72be9d217210c65 Mon Sep 17 00:00:00 2001
From: Petrichor <xiaowenli@selectdb.com>
Date: Fri, 10 Jan 2025 10:34:08 +0800
Subject: [PATCH 096/140] [nereids](feat) implement CleanLabelCommand in
 nereids (#46624)

Issue Number: close https://github.com/apache/doris/issues/42569
---
 .../org/apache/doris/nereids/DorisParser.g4   |  4 +-
 .../apache/doris/load/loadv2/LoadManager.java |  5 +-
 .../nereids/parser/LogicalPlanBuilder.java    |  9 ++
 .../doris/nereids/trees/plans/PlanType.java   |  1 +
 .../commands/clean/CleanLabelCommand.java     | 68 ++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |  5 ++
 .../java/org/apache/doris/qe/DdlExecutor.java |  2 +-
 .../ddl/clean/test_clean_label_nereids.groovy | 88 +++++++++++++++++++
 8 files changed, 175 insertions(+), 7 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/clean/CleanLabelCommand.java
 create mode 100644 regression-test/suites/nereids_p0/ddl/clean/test_clean_label_nereids.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 49392e982490cd4..a1a3ac6d297754e 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -480,6 +480,7 @@ supportedRefreshStatement
 
 supportedCleanStatement
     : CLEAN ALL PROFILE                                                             #cleanAllProfile
+    | CLEAN LABEL label=identifier? (FROM | IN) database=identifier                 #cleanLabel
     ;
 
 unsupportedRefreshStatement
@@ -487,8 +488,7 @@ unsupportedRefreshStatement
     ;
 
 unsupportedCleanStatement
-    : CLEAN LABEL label=identifier? (FROM | IN) database=identifier                 #cleanLabel
-    | CLEAN QUERY STATS ((FOR database=identifier)
+    : CLEAN QUERY STATS ((FOR database=identifier)
         | ((FROM | IN) table=multipartIdentifier))                                  #cleanQueryStats
     | CLEAN ALL QUERY STATS                                                         #cleanAllQueryStats
     ;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java
index d0f06b37fb33367..e8daab69d6f8b3e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java
@@ -19,7 +19,6 @@
 
 import org.apache.doris.analysis.BrokerDesc;
 import org.apache.doris.analysis.CancelLoadStmt;
-import org.apache.doris.analysis.CleanLabelStmt;
 import org.apache.doris.analysis.CompoundPredicate.Operator;
 import org.apache.doris.analysis.InsertStmt;
 import org.apache.doris.analysis.LoadStmt;
@@ -859,9 +858,7 @@ private void checkLabelUsed(long dbId, String label) throws DdlException {
         }
     }
 
-    public void cleanLabel(CleanLabelStmt stmt) throws DdlException {
-        String dbName = stmt.getDb();
-        String label = stmt.getLabel();
+    public void cleanLabel(String dbName, String label) throws DdlException {
         Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(dbName);
         cleanLabelInternal(db.getId(), label, false);
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index af267f92bdf5edf..32f32f4f6c598c1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -98,6 +98,7 @@
 import org.apache.doris.nereids.DorisParser.CancelMTMVTaskContext;
 import org.apache.doris.nereids.DorisParser.CastDataTypeContext;
 import org.apache.doris.nereids.DorisParser.CleanAllProfileContext;
+import org.apache.doris.nereids.DorisParser.CleanLabelContext;
 import org.apache.doris.nereids.DorisParser.CollateContext;
 import org.apache.doris.nereids.DorisParser.ColumnDefContext;
 import org.apache.doris.nereids.DorisParser.ColumnDefsContext;
@@ -617,6 +618,7 @@
 import org.apache.doris.nereids.trees.plans.commands.UnsupportedCommand;
 import org.apache.doris.nereids.trees.plans.commands.UpdateCommand;
 import org.apache.doris.nereids.trees.plans.commands.alter.AlterDatabaseRenameCommand;
+import org.apache.doris.nereids.trees.plans.commands.clean.CleanLabelCommand;
 import org.apache.doris.nereids.trees.plans.commands.info.AddColumnOp;
 import org.apache.doris.nereids.trees.plans.commands.info.AddColumnsOp;
 import org.apache.doris.nereids.trees.plans.commands.info.AddPartitionOp;
@@ -5054,6 +5056,13 @@ public LogicalPlan visitCleanAllProfile(CleanAllProfileContext ctx) {
         return new CleanAllProfileCommand();
     }
 
+    @Override
+    public Object visitCleanLabel(CleanLabelContext ctx) {
+        String label = ctx.label == null ? null : ctx.label.getText();
+        IdentifierContext database = ctx.database;
+        return new CleanLabelCommand(stripQuotes(database.getText()), label);
+    }
+
     @Override
     public LogicalPlan visitShowWhitelist(ShowWhitelistContext ctx) {
         return new ShowWhiteListCommand();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index adbf3720c05c2de..5120bbbf4b2dd38 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -177,6 +177,7 @@ public enum PlanType {
     CREATE_VIEW_COMMAND,
     CLEAN_ALL_PROFILE_COMMAND,
     CLEAN_TRASH_COMMAND,
+    CLEAN_LABEL_COMMAND,
     CREATE_ROLE_COMMAND,
     ALTER_CATALOG_RENAME_COMMAND,
     ALTER_ROLE_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/clean/CleanLabelCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/clean/CleanLabelCommand.java
new file mode 100644
index 000000000000000..49acebca998cd93
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/clean/CleanLabelCommand.java
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands.clean;
+
+import org.apache.doris.analysis.StmtType;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.datasource.InternalCatalog;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.commands.Command;
+import org.apache.doris.nereids.trees.plans.commands.ForwardWithSync;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.base.Strings;
+
+/**
+ * clean label command.
+ */
+public class CleanLabelCommand extends Command implements ForwardWithSync {
+    private final String dbName;
+    private String label;
+
+    public CleanLabelCommand(String dbName, String label) {
+        super(PlanType.CLEAN_LABEL_COMMAND);
+        this.dbName = dbName;
+        this.label = label;
+    }
+
+    @Override
+    public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        label = Strings.nullToEmpty(label);
+        // check auth
+        if (!Env.getCurrentEnv().getAccessManager()
+                .checkDbPriv(ConnectContext.get(), InternalCatalog.INTERNAL_CATALOG_NAME, dbName, PrivPredicate.LOAD)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "LOAD");
+        }
+        Env.getCurrentEnv().getLoadManager().cleanLabel(dbName, label);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitCleanLabelCommand(this, context);
+    }
+
+    @Override
+    public StmtType stmtType() {
+        return StmtType.CLEAN;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 2c926c1de1a9a21..15adde32ffb3eed 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -152,6 +152,7 @@
 import org.apache.doris.nereids.trees.plans.commands.UnsupportedCommand;
 import org.apache.doris.nereids.trees.plans.commands.UpdateCommand;
 import org.apache.doris.nereids.trees.plans.commands.alter.AlterDatabaseRenameCommand;
+import org.apache.doris.nereids.trees.plans.commands.clean.CleanLabelCommand;
 import org.apache.doris.nereids.trees.plans.commands.insert.BatchInsertIntoTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.insert.InsertOverwriteTableCommand;
@@ -565,6 +566,10 @@ default R visitCleanAllProfileCommand(CleanAllProfileCommand cleanAllProfileComm
         return visitCommand(cleanAllProfileCommand, context);
     }
 
+    default R visitCleanLabelCommand(CleanLabelCommand cleanLabelCommand, C context) {
+        return visitCommand(cleanLabelCommand, context);
+    }
+
     default R visitShowDataTypesCommand(ShowDataTypesCommand showDataTypesCommand, C context) {
         return visitCommand(showDataTypesCommand, context);
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java
index 296ba4ac15c7ed2..68fa34071f6f77c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java
@@ -389,7 +389,7 @@ public static void execute(Env env, DdlStmt ddlStmt) throws Exception {
         } else if (ddlStmt instanceof AlterCatalogPropertyStmt) {
             env.getCatalogMgr().alterCatalogProps((AlterCatalogPropertyStmt) ddlStmt);
         } else if (ddlStmt instanceof CleanLabelStmt) {
-            env.getLoadManager().cleanLabel((CleanLabelStmt) ddlStmt);
+            env.getLoadManager().cleanLabel(((CleanLabelStmt) ddlStmt).getDb(), ((CleanLabelStmt) ddlStmt).getLabel());
         } else if (ddlStmt instanceof DropMaterializedViewStmt) {
             env.dropMaterializedView((DropMaterializedViewStmt) ddlStmt);
         } else if (ddlStmt instanceof RefreshCatalogStmt) {
diff --git a/regression-test/suites/nereids_p0/ddl/clean/test_clean_label_nereids.groovy b/regression-test/suites/nereids_p0/ddl/clean/test_clean_label_nereids.groovy
new file mode 100644
index 000000000000000..d751ba8cd9a3b11
--- /dev/null
+++ b/regression-test/suites/nereids_p0/ddl/clean/test_clean_label_nereids.groovy
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_clean_label_nereids") {
+    String dbName = "nereids_clean_label"
+    String tableName1 = "test_table1"
+    String tableName2 = "test_table2"
+    String insertLabel = "label_" + System.currentTimeMillis()
+
+    sql """DROP DATABASE IF EXISTS ${dbName}"""
+    sql """create database ${dbName}"""
+    // Clean up the label in advance
+    sql """clean label from ${dbName}"""
+    sql """use ${dbName}"""
+    sql """
+            CREATE TABLE ${dbName}.${tableName1}(
+                user_id            BIGINT       NOT NULL ,
+                name               VARCHAR(20)           ,
+                age                INT
+            )
+            DUPLICATE KEY(user_id)
+            DISTRIBUTED BY HASH(user_id) BUCKETS 10
+            properties
+            (
+            	"replication_num" = "1"
+            )
+            ;
+        """
+    sql """
+            CREATE TABLE ${dbName}.${tableName2}(
+                user_id            BIGINT       NOT NULL COMMENT "user id",
+                name               VARCHAR(20)           COMMENT "name",
+                age                INT                   COMMENT "age"
+            )
+            DUPLICATE KEY(user_id)
+            DISTRIBUTED BY HASH(user_id) BUCKETS 10
+            properties
+            (
+                "replication_num" = "1"
+            )
+            ;
+        """
+
+    sql """
+        INSERT INTO ${dbName}.${tableName1} (user_id, name, age)
+        VALUES (1, "Emily", 25),
+               (2, "Benjamin", 35),
+               (3, "Olivia", 28),
+               (4, "Alexander", 60),
+               (5, "Ava", 17);
+        """
+
+    sql """
+        insert into ${dbName}.${tableName2} with LABEL ${insertLabel} select * from ${dbName}.${tableName1};
+
+        """
+    sql """
+        INSERT INTO ${dbName}.${tableName2} (user_id, name, age)
+                VALUES (6, "Kevin", 30);
+        INSERT INTO ${dbName}.${tableName2} (user_id, name, age)
+                VALUES (7, "Simba", 31);
+        """
+    //
+    def totalLabel = sql """show load from ${dbName}"""
+    println totalLabel
+    assert totalLabel.size() == 4
+    // clean label
+    checkNereidsExecute("clean label ${insertLabel} from ${dbName};")
+    checkNereidsExecute("clean label  from ${dbName};")
+    def labelResult = sql """show load from ${dbName}"""
+    println labelResult
+    assert labelResult.size() == 0
+    sql """drop database ${dbName}"""
+}
\ No newline at end of file

From 584a256a25274722d22c36eafe57ba3cb8e0dc88 Mon Sep 17 00:00:00 2001
From: 924060929 <lanhuajian@selectdb.com>
Date: Fri, 10 Jan 2025 11:16:13 +0800
Subject: [PATCH 097/140] [opt](coordinator) optimize parallel degree of
 shuffle when use nereids (#44754)

optimize parallel degree of shuffle when use nereids , this pr can fix
some performance rollback when upgrade doris from 1.2 to 2.x/3.x
---
 .../aggregate_function_collect.h              |   8 +-
 .../plans/distribute/DistributePlanner.java   |  24 ++---
 .../worker/job/AbstractUnassignedScanJob.java |  18 ++--
 .../worker/job/LocalShuffleAssignedJob.java   |  13 +--
 .../LocalShuffleBucketJoinAssignedJob.java    |   6 +-
 .../worker/job/UnassignedGatherJob.java       |  16 +--
 .../job/UnassignedScanBucketOlapTableJob.java |  99 +++++++++++++-----
 .../worker/job/UnassignedShuffleJob.java      |  34 +-----
 .../java/org/apache/doris/qe/Coordinator.java |   3 +-
 .../doris/qe/runtime/ThriftPlansBuilder.java  |  21 ++--
 .../test_hdfs_orc_group0_orc_files.out        | Bin 101642 -> 101666 bytes
 .../test_hdfs_orc_group0_orc_files.groovy     |   6 +-
 12 files changed, 127 insertions(+), 121 deletions(-)

diff --git a/be/src/vec/aggregate_functions/aggregate_function_collect.h b/be/src/vec/aggregate_functions/aggregate_function_collect.h
index 2d18a56313f3f91..1b4eadf259de8b7 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_collect.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_collect.h
@@ -263,10 +263,10 @@ struct AggregateFunctionCollectListData<StringRef, HasLimit> {
             }
             max_size = rhs.max_size;
 
-            data->insert_range_from(
-                    *rhs.data, 0,
-                    std::min(assert_cast<size_t, TypeCheckOnRelease::DISABLE>(max_size - size()),
-                             rhs.size()));
+            data->insert_range_from(*rhs.data, 0,
+                                    std::min(assert_cast<size_t, TypeCheckOnRelease::DISABLE>(
+                                                     static_cast<size_t>(max_size - size())),
+                                             rhs.size()));
         } else {
             data->insert_range_from(*rhs.data, 0, rhs.size());
         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributePlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributePlanner.java
index 75a2326236fc9bd..388cef6f0629373 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributePlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributePlanner.java
@@ -46,7 +46,6 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.LinkedHashMultimap;
 import com.google.common.collect.ListMultimap;
-import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.SetMultimap;
 import org.apache.logging.log4j.LogManager;
@@ -136,6 +135,16 @@ private FragmentIdMapping<DistributedPlan> linkPlans(FragmentIdMapping<Distribut
                         link.getKey(),
                         enableShareHashTableForBroadcastJoin
                 );
+                for (Entry<DataSink, List<AssignedJob>> kv :
+                        ((PipelineDistributedPlan) link.getValue()).getDestinations().entrySet()) {
+                    if (kv.getValue().isEmpty()) {
+                        int sourceFragmentId = link.getValue().getFragmentJob().getFragment().getFragmentId().asInt();
+                        String msg = "Invalid plan which exchange not contains receiver, "
+                                + "exchange id: " + kv.getKey().getExchNodeId().asInt()
+                                + ", source fragmentId: " + sourceFragmentId;
+                        throw new IllegalStateException(msg);
+                    }
+                }
             }
         }
         return plans;
@@ -184,7 +193,7 @@ private List<AssignedJob> filterInstancesWhichCanReceiveDataFromRemote(
         boolean useLocalShuffle = receiverPlan.getInstanceJobs().stream()
                 .anyMatch(LocalShuffleAssignedJob.class::isInstance);
         if (useLocalShuffle) {
-            return getFirstInstancePerShareScan(receiverPlan);
+            return getFirstInstancePerWorker(receiverPlan.getInstanceJobs());
         } else if (enableShareHashTableForBroadcastJoin && linkNode.isRightChildOfBroadcastHashJoin()) {
             return getFirstInstancePerWorker(receiverPlan.getInstanceJobs());
         } else {
@@ -221,17 +230,6 @@ private List<AssignedJob> sortDestinationInstancesByBuckets(
         return Arrays.asList(instances);
     }
 
-    private List<AssignedJob> getFirstInstancePerShareScan(PipelineDistributedPlan plan) {
-        List<AssignedJob> canReceiveDataFromRemote = Lists.newArrayListWithCapacity(plan.getInstanceJobs().size());
-        for (AssignedJob instanceJob : plan.getInstanceJobs()) {
-            LocalShuffleAssignedJob localShuffleJob = (LocalShuffleAssignedJob) instanceJob;
-            if (!localShuffleJob.receiveDataFromLocal) {
-                canReceiveDataFromRemote.add(localShuffleJob);
-            }
-        }
-        return canReceiveDataFromRemote;
-    }
-
     private List<AssignedJob> getFirstInstancePerWorker(List<AssignedJob> instances) {
         Map<DistributedPlanWorker, AssignedJob> firstInstancePerWorker = Maps.newLinkedHashMap();
         for (AssignedJob instance : instances) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AbstractUnassignedScanJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AbstractUnassignedScanJob.java
index d2fbb9905e1aeae..37d665adcc4d707 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AbstractUnassignedScanJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AbstractUnassignedScanJob.java
@@ -91,7 +91,7 @@ protected List<AssignedJob> insideMachineParallelization(
 
             // now we should compute how many instances to process the data,
             // for example: two instances
-            int instanceNum = degreeOfParallelism(scanSourceMaxParallel);
+            int instanceNum = degreeOfParallelism(scanSourceMaxParallel, useLocalShuffleToAddParallel);
 
             if (useLocalShuffleToAddParallel) {
                 assignLocalShuffleJobs(scanSource, instanceNum, instances, context, worker);
@@ -129,7 +129,7 @@ protected void assignedDefaultJobs(ScanSource scanSource, int instanceNum, List<
     protected void assignLocalShuffleJobs(ScanSource scanSource, int instanceNum, List<AssignedJob> instances,
             ConnectContext context, DistributedPlanWorker worker) {
         // only generate one instance to scan all data, in this step
-        List<ScanSource> instanceToScanRanges = scanSource.parallelize(scanNodes, 1);
+        List<ScanSource> assignedJoinBuckets = scanSource.parallelize(scanNodes, instanceNum);
 
         // when data not big, but aggregation too slow, we will use 1 instance to scan data,
         // and use more instances (to ***add parallel***) to process aggregate.
@@ -144,7 +144,7 @@ protected void assignLocalShuffleJobs(ScanSource scanSource, int instanceNum, Li
         // |(share scan node, instance1 will scan all data and local shuffle to other local instances       |
         // |                           to parallel compute this data)                                       |
         // +------------------------------------------------------------------------------------------------+
-        ScanSource shareScanSource = instanceToScanRanges.get(0);
+        ScanSource shareScanSource = assignedJoinBuckets.get(0);
 
         // one scan range generate multiple instances,
         // different instances reference the same scan source
@@ -152,15 +152,15 @@ protected void assignLocalShuffleJobs(ScanSource scanSource, int instanceNum, Li
         ScanSource emptyShareScanSource = shareScanSource.newEmpty();
         for (int i = 0; i < instanceNum; i++) {
             LocalShuffleAssignedJob instance = new LocalShuffleAssignedJob(
-                    instances.size(), shareScanId, i > 0,
-                    context.nextInstanceId(), this, worker,
-                    i == 0 ? shareScanSource : emptyShareScanSource
+                    instances.size(), shareScanId, context.nextInstanceId(), this, worker,
+                    // only first instance need to scan data
+                    i == 0 ? scanSource : emptyShareScanSource
             );
             instances.add(instance);
         }
     }
 
-    protected int degreeOfParallelism(int maxParallel) {
+    protected int degreeOfParallelism(int maxParallel, boolean useLocalShuffleToAddParallel) {
         Preconditions.checkArgument(maxParallel > 0, "maxParallel must be positive");
         if (!fragment.getDataPartition().isPartitioned()) {
             return 1;
@@ -179,6 +179,10 @@ protected int degreeOfParallelism(int maxParallel) {
             }
         }
 
+        if (useLocalShuffleToAddParallel) {
+            return Math.max(fragment.getParallelExecNum(), 1);
+        }
+
         // the scan instance num should not larger than the tablets num
         return Math.min(maxParallel, Math.max(fragment.getParallelExecNum(), 1));
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/LocalShuffleAssignedJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/LocalShuffleAssignedJob.java
index 2ba269a5a7b89fb..9bb7c61ffafe45f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/LocalShuffleAssignedJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/LocalShuffleAssignedJob.java
@@ -31,28 +31,17 @@
  */
 public class LocalShuffleAssignedJob extends StaticAssignedJob {
     public final int shareScanId;
-    public final boolean receiveDataFromLocal;
 
     public LocalShuffleAssignedJob(
-            int indexInUnassignedJob, int shareScanId, boolean receiveDataFromLocal, TUniqueId instanceId,
+            int indexInUnassignedJob, int shareScanId, TUniqueId instanceId,
             UnassignedJob unassignedJob,
             DistributedPlanWorker worker, ScanSource scanSource) {
         super(indexInUnassignedJob, instanceId, unassignedJob, worker, scanSource);
         this.shareScanId = shareScanId;
-        this.receiveDataFromLocal = receiveDataFromLocal;
     }
 
     @Override
     protected Map<String, String> extraInfo() {
         return ImmutableMap.of("shareScanIndex", String.valueOf(shareScanId));
     }
-
-    @Override
-    protected String formatScanSourceString() {
-        if (receiveDataFromLocal) {
-            return "read data from first instance of " + getAssignedWorker();
-        } else {
-            return super.formatScanSourceString();
-        }
-    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/LocalShuffleBucketJoinAssignedJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/LocalShuffleBucketJoinAssignedJob.java
index 443acb50d78c787..9090bf98b364c26 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/LocalShuffleBucketJoinAssignedJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/LocalShuffleBucketJoinAssignedJob.java
@@ -30,11 +30,11 @@ public class LocalShuffleBucketJoinAssignedJob extends LocalShuffleAssignedJob {
     private volatile Set<Integer> assignedJoinBucketIndexes;
 
     public LocalShuffleBucketJoinAssignedJob(
-            int indexInUnassignedJob, int shareScanId, boolean receiveDataFromLocal,
+            int indexInUnassignedJob, int shareScanId,
             TUniqueId instanceId, UnassignedJob unassignedJob,
             DistributedPlanWorker worker, ScanSource scanSource,
             Set<Integer> assignedJoinBucketIndexes) {
-        super(indexInUnassignedJob, shareScanId, receiveDataFromLocal, instanceId, unassignedJob, worker, scanSource);
+        super(indexInUnassignedJob, shareScanId, instanceId, unassignedJob, worker, scanSource);
         this.assignedJoinBucketIndexes = Utils.fastToImmutableSet(assignedJoinBucketIndexes);
     }
 
@@ -42,7 +42,7 @@ public Set<Integer> getAssignedJoinBucketIndexes() {
         return assignedJoinBucketIndexes;
     }
 
-    public void addAssignedJoinBucketIndexes(Set<Integer> joinBucketIndexes) {
+    public synchronized void addAssignedJoinBucketIndexes(Set<Integer> joinBucketIndexes) {
         this.assignedJoinBucketIndexes = ImmutableSet.<Integer>builder()
                 .addAll(assignedJoinBucketIndexes)
                 .addAll(joinBucketIndexes)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedGatherJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedGatherJob.java
index 830342514bd387e..6fe9fcfb439b501 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedGatherJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedGatherJob.java
@@ -32,7 +32,7 @@
 
 /** UnassignedGatherJob */
 public class UnassignedGatherJob extends AbstractUnassignedJob {
-    private boolean useLocalShuffleToAddParallel;
+    private boolean useSerialSource;
 
     public UnassignedGatherJob(
             StatementContext statementContext, PlanFragment fragment,
@@ -44,24 +44,24 @@ public UnassignedGatherJob(
     public List<AssignedJob> computeAssignedJobs(
             DistributeContext distributeContext, ListMultimap<ExchangeNode, AssignedJob> inputJobs) {
         ConnectContext connectContext = statementContext.getConnectContext();
-        useLocalShuffleToAddParallel = fragment.useSerialSource(connectContext);
+        useSerialSource = fragment.useSerialSource(connectContext);
 
         int expectInstanceNum = degreeOfParallelism();
 
         DistributedPlanWorker selectedWorker = distributeContext.selectedWorkers.tryToSelectRandomUsedWorker();
-        if (useLocalShuffleToAddParallel) {
+        if (useSerialSource) {
+            // Using serial source means a serial source operator will be used in this fragment (e.g. data will be
+            // shuffled to only 1 exchange operator) and then split by followed local exchanger
             ImmutableList.Builder<AssignedJob> instances = ImmutableList.builder();
-
             DefaultScanSource shareScan = new DefaultScanSource(ImmutableMap.of());
             LocalShuffleAssignedJob receiveDataFromRemote = new LocalShuffleAssignedJob(
-                    0, 0, false,
+                    0, 0,
                     connectContext.nextInstanceId(), this, selectedWorker, shareScan);
 
             instances.add(receiveDataFromRemote);
             for (int i = 1; i < expectInstanceNum; ++i) {
                 LocalShuffleAssignedJob receiveDataFromLocal = new LocalShuffleAssignedJob(
-                        i, 0, true,
-                        connectContext.nextInstanceId(), this, selectedWorker, shareScan);
+                        i, 0, connectContext.nextInstanceId(), this, selectedWorker, shareScan);
                 instances.add(receiveDataFromLocal);
             }
             return instances.build();
@@ -76,6 +76,6 @@ selectedWorker, new DefaultScanSource(ImmutableMap.of())
     }
 
     protected int degreeOfParallelism() {
-        return useLocalShuffleToAddParallel ? fragment.getParallelExecNum() : 1;
+        return useSerialSource ? Math.max(1, fragment.getParallelExecNum()) : 1;
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java
index f90fe7ea6e2c211..88612640d508dca 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java
@@ -38,12 +38,16 @@
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ListMultimap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -184,13 +188,23 @@ protected void assignLocalShuffleJobs(ScanSource scanSource, int instanceNum, Li
             Set<Integer> assignedJoinBuckets
                     = ((BucketScanSource) assignJoinBuckets.get(i)).bucketIndexToScanNodeToTablets.keySet();
             LocalShuffleBucketJoinAssignedJob instance = new LocalShuffleBucketJoinAssignedJob(
-                    instances.size(), shareScanId, i > 0,
-                    context.nextInstanceId(), this, worker,
+                    instances.size(), shareScanId, context.nextInstanceId(),
+                    this, worker,
                     i == 0 ? shareScanSource : emptyShareScanSource,
                     Utils.fastToImmutableSet(assignedJoinBuckets)
             );
             instances.add(instance);
         }
+
+        for (int i = assignJoinBuckets.size(); i < instanceNum; ++i) {
+            LocalShuffleBucketJoinAssignedJob instance = new LocalShuffleBucketJoinAssignedJob(
+                    instances.size(), shareScanId, context.nextInstanceId(),
+                    this, worker, emptyShareScanSource,
+                    // these instance not need to join, because no any bucket assign to it
+                    ImmutableSet.of()
+            );
+            instances.add(instance);
+        }
     }
 
     private boolean shouldFillUpInstances(List<HashJoinNode> hashJoinNodes) {
@@ -224,10 +238,21 @@ private List<AssignedJob> fillUpInstances(List<AssignedJob> instances) {
                 olapScanNode, randomPartition, missingBucketIndexes);
 
         boolean useLocalShuffle = instances.stream().anyMatch(LocalShuffleAssignedJob.class::isInstance);
+        Multimap<DistributedPlanWorker, AssignedJob> workerToAssignedJobs = ArrayListMultimap.create();
+        int maxNumInstancePerWorker = 1;
+        if (useLocalShuffle) {
+            for (AssignedJob instance : instances) {
+                workerToAssignedJobs.put(instance.getAssignedWorker(), instance);
+            }
+            for (Collection<AssignedJob> instanceList : workerToAssignedJobs.asMap().values()) {
+                maxNumInstancePerWorker = Math.max(maxNumInstancePerWorker, instanceList.size());
+            }
+        }
+
         List<AssignedJob> newInstances = new ArrayList<>(instances);
+
         for (Entry<DistributedPlanWorker, Collection<Integer>> workerToBuckets : missingBuckets.asMap().entrySet()) {
             Map<Integer, Map<ScanNode, ScanRanges>> scanEmptyBuckets = Maps.newLinkedHashMap();
-            Set<Integer> assignedJoinBuckets = Utils.fastToImmutableSet(workerToBuckets.getValue());
             for (Integer bucketIndex : workerToBuckets.getValue()) {
                 Map<ScanNode, ScanRanges> scanTableWithEmptyData = Maps.newLinkedHashMap();
                 for (ScanNode scanNode : scanNodes) {
@@ -236,42 +261,62 @@ private List<AssignedJob> fillUpInstances(List<AssignedJob> instances) {
                 scanEmptyBuckets.put(bucketIndex, scanTableWithEmptyData);
             }
 
-            AssignedJob fillUpInstance = null;
             DistributedPlanWorker worker = workerToBuckets.getKey();
             BucketScanSource scanSource = new BucketScanSource(scanEmptyBuckets);
             if (useLocalShuffle) {
-                // when use local shuffle, we should ensure every backend only process one instance!
-                // so here we should try to merge the missing buckets into exist instances
-                boolean mergedBucketsInSameWorkerInstance = false;
-                for (AssignedJob newInstance : newInstances) {
-                    if (newInstance.getAssignedWorker().equals(worker)) {
-                        BucketScanSource bucketScanSource = (BucketScanSource) newInstance.getScanSource();
-                        bucketScanSource.bucketIndexToScanNodeToTablets.putAll(scanEmptyBuckets);
-                        mergedBucketsInSameWorkerInstance = true;
-
-                        LocalShuffleBucketJoinAssignedJob instance = (LocalShuffleBucketJoinAssignedJob) newInstance;
-                        instance.addAssignedJoinBucketIndexes(assignedJoinBuckets);
-                    }
+                List<AssignedJob> sameWorkerInstances = (List) workerToAssignedJobs.get(worker);
+                if (sameWorkerInstances.isEmpty()) {
+                    sameWorkerInstances = fillUpEmptyInstances(
+                            maxNumInstancePerWorker, scanSource, worker, newInstances, context);
                 }
-                if (!mergedBucketsInSameWorkerInstance) {
-                    fillUpInstance = new LocalShuffleBucketJoinAssignedJob(
-                            newInstances.size(), shareScanIdGenerator.getAndIncrement(),
-                            false, context.nextInstanceId(), this, worker, scanSource,
-                            assignedJoinBuckets
-                    );
+
+                LocalShuffleBucketJoinAssignedJob firstInstance
+                        = (LocalShuffleBucketJoinAssignedJob ) sameWorkerInstances.get(0);
+                BucketScanSource firstInstanceScanSource
+                        = (BucketScanSource) firstInstance.getScanSource();
+                firstInstanceScanSource.bucketIndexToScanNodeToTablets.putAll(scanEmptyBuckets);
+
+                Iterator<Integer> assignedJoinBuckets = new LinkedHashSet<>(workerToBuckets.getValue()).iterator();
+                // make sure the first instance must be assigned some buckets:
+                // if the first instance assigned some buckets, we start assign empty
+                // bucket for second instance for balance, or else assign for first instance
+                int index = firstInstance.getAssignedJoinBucketIndexes().isEmpty() ? -1 : 0;
+                while (assignedJoinBuckets.hasNext()) {
+                    Integer bucketIndex = assignedJoinBuckets.next();
+                    assignedJoinBuckets.remove();
+
+                    index = (index + 1) % sameWorkerInstances.size();
+                    LocalShuffleBucketJoinAssignedJob instance
+                            = (LocalShuffleBucketJoinAssignedJob) sameWorkerInstances.get(index);
+                    instance.addAssignedJoinBucketIndexes(ImmutableSet.of(bucketIndex));
                 }
             } else {
-                fillUpInstance = assignWorkerAndDataSources(
+                newInstances.add(assignWorkerAndDataSources(
                         newInstances.size(), context.nextInstanceId(), worker, scanSource
-                );
-            }
-            if (fillUpInstance != null) {
-                newInstances.add(fillUpInstance);
+                ));
             }
         }
         return newInstances;
     }
 
+    private List<AssignedJob> fillUpEmptyInstances(
+            int maxNumInstancePerWorker, BucketScanSource scanSource, DistributedPlanWorker worker,
+            List<AssignedJob> existsInstances, ConnectContext context) {
+        int shareScanId = shareScanIdGenerator.getAndIncrement();
+        List<AssignedJob> newInstances = new ArrayList<>(maxNumInstancePerWorker);
+        for (int i = 0; i < maxNumInstancePerWorker; i++) {
+            LocalShuffleBucketJoinAssignedJob newInstance = new LocalShuffleBucketJoinAssignedJob(
+                    existsInstances.size(), shareScanId,
+                    context.nextInstanceId(), this, worker,
+                    scanSource.newEmpty(),
+                    ImmutableSet.of()
+            );
+            existsInstances.add(newInstance);
+            newInstances.add(newInstance);
+        }
+        return newInstances;
+    }
+
     private int fullBucketNum() {
         for (ScanNode scanNode : scanNodes) {
             if (scanNode instanceof OlapScanNode) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedShuffleJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedShuffleJob.java
index 3d937bfb35dbf9f..d8eac65cded6c5b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedShuffleJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedShuffleJob.java
@@ -17,13 +17,11 @@
 
 package org.apache.doris.nereids.trees.plans.distribute.worker.job;
 
-import org.apache.doris.common.Pair;
 import org.apache.doris.nereids.StatementContext;
 import org.apache.doris.nereids.trees.plans.distribute.DistributeContext;
 import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker;
 import org.apache.doris.planner.ExchangeNode;
 import org.apache.doris.planner.PlanFragment;
-import org.apache.doris.planner.PlanNode;
 import org.apache.doris.qe.ConnectContext;
 
 import com.google.common.collect.ArrayListMultimap;
@@ -43,7 +41,7 @@
 
 /** UnassignedShuffleJob */
 public class UnassignedShuffleJob extends AbstractUnassignedJob {
-    private boolean useLocalShuffleToAddParallel;
+    private boolean useSerialSource;
 
     public UnassignedShuffleJob(
             StatementContext statementContext, PlanFragment fragment,
@@ -54,7 +52,7 @@ public UnassignedShuffleJob(
     @Override
     public List<AssignedJob> computeAssignedJobs(
             DistributeContext distributeContext, ListMultimap<ExchangeNode, AssignedJob> inputJobs) {
-        useLocalShuffleToAddParallel = fragment.useSerialSource(statementContext.getConnectContext());
+        useSerialSource = fragment.useSerialSource(statementContext.getConnectContext());
 
         int expectInstanceNum = degreeOfParallelism();
         List<AssignedJob> biggestParallelChildFragment = getInstancesOfBiggestParallelChildFragment(inputJobs);
@@ -83,18 +81,10 @@ public List<AssignedJob> computeAssignedJobs(
     protected int degreeOfParallelism() {
         // TODO: check we use nested loop join do right outer / semi / anti join,
         //       we should add an exchange node with gather distribute under the nested loop join
-
         int expectInstanceNum = -1;
         if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable() != null) {
             expectInstanceNum = ConnectContext.get().getSessionVariable().getExchangeInstanceParallel();
         }
-
-        // TODO: check nested loop join do right outer / semi / anti join
-        PlanNode leftMostNode = findLeftmostNode(fragment.getPlanRoot()).second;
-        // when we use nested loop join do right outer / semi / anti join, the instance must be 1.
-        if (leftMostNode.getNumInstances() == 1) {
-            expectInstanceNum = 1;
-        }
         return expectInstanceNum;
     }
 
@@ -116,7 +106,7 @@ private List<AssignedJob> getInstancesOfBiggestParallelChildFragment(
     private List<AssignedJob> buildInstances(
             int instanceNum, Function<Integer, DistributedPlanWorker> workerSelector) {
         ConnectContext connectContext = statementContext.getConnectContext();
-        if (useLocalShuffleToAddParallel) {
+        if (useSerialSource) {
             return buildInstancesWithLocalShuffle(instanceNum, workerSelector, connectContext);
         } else {
             return buildInstancesWithoutLocalShuffle(instanceNum, workerSelector, connectContext);
@@ -150,17 +140,13 @@ private List<AssignedJob> buildInstancesWithLocalShuffle(
         for (Entry<DistributedPlanWorker, Collection<Integer>> kv : workerToInstanceIds.asMap().entrySet()) {
             DistributedPlanWorker worker = kv.getKey();
             Collection<Integer> indexesInFragment = kv.getValue();
-
             DefaultScanSource shareScanSource = new DefaultScanSource(ImmutableMap.of());
-
-            boolean receiveDataFromLocal = false;
             for (Integer indexInFragment : indexesInFragment) {
                 LocalShuffleAssignedJob instance = new LocalShuffleAssignedJob(
-                        indexInFragment, shareScanId, receiveDataFromLocal, connectContext.nextInstanceId(),
+                        indexInFragment, shareScanId, connectContext.nextInstanceId(),
                         this, worker, shareScanSource
                 );
                 instances.add(instance);
-                receiveDataFromLocal = true;
             }
             shareScanId++;
         }
@@ -176,16 +162,4 @@ private List<DistributedPlanWorker> distinctShuffleWorkers(List<AssignedJob> ins
         Collections.shuffle(candidateWorkers);
         return candidateWorkers;
     }
-
-    // Returns the id of the leftmost node of any of the gives types in 'plan_root',
-    // or INVALID_PLAN_NODE_ID if no such node present.
-    private Pair<PlanNode, PlanNode> findLeftmostNode(PlanNode plan) {
-        PlanNode childPlan = plan;
-        PlanNode fatherPlan = null;
-        while (childPlan.getChildren().size() != 0 && !(childPlan instanceof ExchangeNode)) {
-            fatherPlan = childPlan;
-            childPlan = childPlan.getChild(0);
-        }
-        return Pair.of(fatherPlan, childPlan);
-    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index 3bf5c44d564065f..09a9a857f33a3a5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -1809,7 +1809,8 @@ protected void computeFragmentHosts() throws Exception {
                     exchangeInstances = ConnectContext.get().getSessionVariable().getExchangeInstanceParallel();
                 }
                 // when we use nested loop join do right outer / semi / anti join, the instance must be 1.
-                if (leftMostNode.getNumInstances() == 1) {
+                boolean isNereids = context != null && context.getState().isNereids();
+                if (!isNereids && leftMostNode.getNumInstances() == 1) {
                     exchangeInstances = 1;
                 }
                 // Using serial source means a serial source operator will be used in this fragment (e.g. data will be
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java
index 54bc0b24d3e8217..0caca2f47c15326 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java
@@ -419,21 +419,18 @@ private static TPipelineInstanceParams instanceToThrift(
         if (runtimeFiltersThriftBuilder.isMergeRuntimeFilterInstance(instance)) {
             runtimeFiltersThriftBuilder.setRuntimeFilterThriftParams(runtimeFilterParams);
         }
+        boolean isLocalShuffle = instance instanceof LocalShuffleAssignedJob;
+        if (isLocalShuffle) {
+            // a fragment in a backend only enable local shuffle once for the first local shuffle instance,
+            // because we just skip set scan params for LocalShuffleAssignedJob.receiveDataFromLocal == true
+            ignoreDataDistribution(currentFragmentParam);
+        }
         return instanceParam;
     }
 
     private static void setScanSourceParam(
             TPipelineFragmentParams currentFragmentParam, AssignedJob instance,
             TPipelineInstanceParams instanceParams) {
-
-        boolean isLocalShuffle = instance instanceof LocalShuffleAssignedJob;
-        if (isLocalShuffle && ((LocalShuffleAssignedJob) instance).receiveDataFromLocal) {
-            // save thrift rpc message size, don't need perNodeScanRanges,
-            // but the perNodeScanRanges is required rpc field
-            instanceParams.setPerNodeScanRanges(Maps.newLinkedHashMap());
-            return;
-        }
-
         ScanSource scanSource = instance.getScanSource();
         PerNodeScanParams scanParams;
         if (scanSource instanceof BucketScanSource) {
@@ -443,12 +440,6 @@ private static void setScanSourceParam(
         }
         // perNodeScanRanges is required
         instanceParams.setPerNodeScanRanges(scanParams.perNodeScanRanges);
-
-        if (isLocalShuffle) {
-            // a fragment in a backend only enable local shuffle once for the first local shuffle instance,
-            // because we just skip set scan params for LocalShuffleAssignedJob.receiveDataFromLocal == true
-            ignoreDataDistribution(currentFragmentParam);
-        }
     }
 
     // local shuffle has two functions:
diff --git a/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.out b/regression-test/data/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.out
index 01158a2fb603f254f6d13e2651976d21f6929639..cb2f86d7ba47e4e1274e0b384cab58ba6795876e 100644
GIT binary patch
delta 1254
zcmZXUF^<+i3`Rw>QuY#@;74Q6*dCvQ6KsJ*vkihe1szA&l8$QWxE`M!@4us@d$N=#
z=Vd0pULJnGJp6tAx|3};reQ<6I@p<eH*Q_0#B_>nJYDD$aWC>|>=<p|zy0{~NQ}pN
zvwj!3fKMVNipSNZL_t1xm_Vzkv%i@JH}KhnMPVA<%M_F=0ag)qzl(-~&mv{nVWQc~
z6qF{wYMO)YCS&l~G$(Cr&UT2BLNqa2M59IGBlmCv?<bQ{M{iTB-brb~Xf^d{e3%4Z
z?6Q(|M{gTcy^~Ue(Yt8B4UNWS8uQxQbfCA9RPUrTVf1dgjTwzggp+e!$(wAf-YR#{
zp1lpUn1rJ7Vba#Fy(cMptB&fO6f0rZwYT(WTq4FvK7Y8m4_pA&?$tXfO&F~!X*4dA
z;3!{AiVMI>y?Q652%}XrBXNmL_iJyQ8T6KE)jKIniqT?HipGb@8ee-m;R3KISMQ`0
zVN{|$F0YPgTqfLy%ggz|1z`Lks&`VFFj`mAXj~%1zl!1?!$MTK<#S+~0ISKPahVK9
b`P>`-AM_qms&`U|Wd3}5{`d6!;m^naX*mG{

delta 1230
zcmZY9J#O4E5CveQF~D5HC)i+qIV5}@18Lj@ap6iux*Wk}x>QN$>-h~uEJUf|!(j1&
zdOOme_lMu_5C8vt-;AT&ge@$(bmT6g8bVr|TLha&QFZ>tqQ0lz!Z0a6f4n|_Wl5mb
z(7WS~e0GAhD0NR~CODg6v_QDU9cqMphI;Kqn)V$U36NrPbUHVmP9v(wCr3oQ$mUHa
zlqn&c3uMN3NCo)}S>qxqF`Z25;9Q)V?T)4!dCaD_L)8)*MJPtqx~GINS)iWvPN<>F
z%Y}N^!i{IuBf$|&4vvzb-EBfl?OL^p^}_D6>X8z{WPvQ}-7myy*Q(XY3oFg0M@k2i
zZGj@|olvD~)f$xxlZ~onPYGeNI6do~PQ(758#2PyM@P|{=!|d}qoXr6bvnY>-;0C4
zthkP%!DLDX=fYTayI)5X+k0TP@sgQi=_N+bY$K(E$>KzIJDrN{JvciP*m)T(GfAdY
zNRIogLRsuo+J@adGfrdUbwE+^V?;<%mzMGV+*?-0>Rz1jJ;B50i-Fr_0^@ur`RS(O
h+3bEHRjlsC8Q&8;d^%EYnc&E;x0jE%moI-GKLKds`yK!Q

diff --git a/regression-test/suites/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.groovy b/regression-test/suites/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.groovy
index 924ceca4204fef7..bf2929ee5b40855 100644
--- a/regression-test/suites/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.groovy
+++ b/regression-test/suites/external_table_p0/tvf/orc_tvf/test_hdfs_orc_group0_orc_files.groovy
@@ -174,7 +174,11 @@ suite("test_hdfs_orc_group0_orc_files","external,hive,tvf,external_docker") {
             order_qt_test_25 """ select * from HDFS(
                         "uri" = "${uri}",
                         "hadoop.username" = "${hdfsUserName}",
-                        "format" = "orc") order by _col0 DESC limit 100; """
+                        "format" = "orc")
+                        where _col0 is not null and _col1 is not null and _col2 is not null and _col3 is not null
+                              and _col4 is not null and _col5 is not null and _col6 is not null
+                        order by _col0 DESC, _col1 DESC, _col2 DESC, _col3 DESC, _col4 DESC, _col5 DESC, _col6 DESC
+                        limit 100; """
 
 
             uri = "${defaultFS}" + "/user/doris/tvf_data/test_hdfs_orc/group0/TestVectorOrcFile.testLzo.orc"

From 977b334119f8f5f1c3b514657d15ed9de72078b9 Mon Sep 17 00:00:00 2001
From: shee <13843187+qzsee@users.noreply.github.com>
Date: Fri, 10 Jan 2025 11:35:42 +0800
Subject: [PATCH 098/140]  [fix][nereids] fix split count distinct for null
 can't join (#46563)

### What problem does this PR solve?

related pr: #45209

If split cout distinct and group by are used, inner join will be used to
link the results, but if the group by column has a null value, inner
join will not be linked, so we use `NullSafeEqual` as the link condition

---------

Co-authored-by: garenshi <garenshi@tencent.com>
---
 .../nereids/rules/rewrite/SplitMultiDistinct.java   |  6 +++---
 .../rules/rewrite/SplitMultiDistinctTest.java       |  4 ++++
 .../distinct_split/disitinct_split.out              | 13 ++++++++-----
 .../distinct_split/disitinct_split.groovy           |  8 +++++++-
 4 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SplitMultiDistinct.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SplitMultiDistinct.java
index 56df1485a83829c..6adb3270b5b30c7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SplitMultiDistinct.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SplitMultiDistinct.java
@@ -24,9 +24,9 @@
 import org.apache.doris.nereids.trees.copier.DeepCopierContext;
 import org.apache.doris.nereids.trees.copier.LogicalPlanDeepCopier;
 import org.apache.doris.nereids.trees.expressions.Alias;
-import org.apache.doris.nereids.trees.expressions.EqualTo;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.NullSafeEqual;
 import org.apache.doris.nereids.trees.expressions.OrderExpression;
 import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
@@ -273,7 +273,7 @@ private static LogicalJoin<Plan, Plan> constructJoin(List<LogicalAggregate<Plan>
             List<Slot> rightSlots = newAggs.get(1).getOutput();
             List<Expression> hashConditions = new ArrayList<>();
             for (int i = 0; i < len; ++i) {
-                hashConditions.add(new EqualTo(leftSlots.get(i), rightSlots.get(i)));
+                hashConditions.add(new NullSafeEqual(leftSlots.get(i), rightSlots.get(i)));
             }
             join = new LogicalJoin<>(JoinType.INNER_JOIN, hashConditions, newAggs.get(0), newAggs.get(1), null);
             for (int j = 2; j < newAggs.size(); ++j) {
@@ -281,7 +281,7 @@ private static LogicalJoin<Plan, Plan> constructJoin(List<LogicalAggregate<Plan>
                 List<Slot> belowRightSlots = newAggs.get(j).getOutput();
                 List<Expression> aboveHashConditions = new ArrayList<>();
                 for (int i = 0; i < len; ++i) {
-                    aboveHashConditions.add(new EqualTo(belowJoinSlots.get(i), belowRightSlots.get(i)));
+                    aboveHashConditions.add(new NullSafeEqual(belowJoinSlots.get(i), belowRightSlots.get(i)));
                 }
                 join = new LogicalJoin<>(JoinType.INNER_JOIN, aboveHashConditions, join, newAggs.get(j), null);
             }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SplitMultiDistinctTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SplitMultiDistinctTest.java
index 074135695a1a316..ebc6331506a5d35 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SplitMultiDistinctTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SplitMultiDistinctTest.java
@@ -17,6 +17,8 @@
 
 package org.apache.doris.nereids.rules.rewrite;
 
+import org.apache.doris.nereids.trees.expressions.NullSafeEqual;
+import org.apache.doris.nereids.trees.plans.JoinType;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.util.MatchingUtils;
 import org.apache.doris.nereids.util.MemoPatternMatchSupported;
@@ -180,6 +182,8 @@ void countMultiColumnsWithGby() {
                                                                     physicalHashAggregate(
                                                                             physicalDistribute(
                                                                                     physicalHashAggregate(any()))))
+                                                    ).when(join ->
+                                                        join.getJoinType() == JoinType.INNER_JOIN && join.getHashJoinConjuncts().get(0) instanceof NullSafeEqual
                                                     )
                                             )
                                     )
diff --git a/regression-test/data/nereids_rules_p0/distinct_split/disitinct_split.out b/regression-test/data/nereids_rules_p0/distinct_split/disitinct_split.out
index 2a1dd6fd9d6705a..a0aff0a9a191270 100644
--- a/regression-test/data/nereids_rules_p0/distinct_split/disitinct_split.out
+++ b/regression-test/data/nereids_rules_p0/distinct_split/disitinct_split.out
@@ -380,7 +380,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --PhysicalCteProducer ( cteId=CTEId#0 )
 ----PhysicalOlapScan[test_distinct_multi]
 --PhysicalResultSink
-----hashJoin[INNER_JOIN] hashCondition=((.d = .d)) otherCondition=()
+----hashJoin[INNER_JOIN] hashCondition=((.d <=> .d)) otherCondition=()
 ------hashAgg[DISTINCT_LOCAL]
 --------hashAgg[GLOBAL]
 ----------hashAgg[LOCAL]
@@ -415,9 +415,9 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --PhysicalCteProducer ( cteId=CTEId#0 )
 ----PhysicalOlapScan[test_distinct_multi]
 --PhysicalResultSink
-----hashJoin[INNER_JOIN] hashCondition=((.d = .d)) otherCondition=()
-------hashJoin[INNER_JOIN] hashCondition=((.d = .d)) otherCondition=()
---------hashJoin[INNER_JOIN] hashCondition=((.d = .d)) otherCondition=()
+----hashJoin[INNER_JOIN] hashCondition=((.d <=> .d)) otherCondition=()
+------hashJoin[INNER_JOIN] hashCondition=((.d <=> .d)) otherCondition=()
+--------hashJoin[INNER_JOIN] hashCondition=((.d <=> .d)) otherCondition=()
 ----------hashAgg[DISTINCT_LOCAL]
 ------------hashAgg[GLOBAL]
 --------------hashAgg[LOCAL]
@@ -463,7 +463,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --PhysicalResultSink
 ----hashAgg[GLOBAL]
 ------hashAgg[LOCAL]
---------hashJoin[INNER_JOIN] hashCondition=((.c = .c)) otherCondition=()
+--------hashJoin[INNER_JOIN] hashCondition=((.c <=> .c)) otherCondition=()
 ----------hashAgg[DISTINCT_LOCAL]
 ------------hashAgg[GLOBAL]
 --------------hashAgg[LOCAL]
@@ -498,3 +498,6 @@ PhysicalResultSink
 ------PhysicalRepeat
 --------PhysicalOlapScan[test_distinct_multi]
 
+-- !null_hash --
+1	\N	0	0.0
+
diff --git a/regression-test/suites/nereids_rules_p0/distinct_split/disitinct_split.groovy b/regression-test/suites/nereids_rules_p0/distinct_split/disitinct_split.groovy
index 02812b269a33ebd..9efe6ce487a4f83 100644
--- a/regression-test/suites/nereids_rules_p0/distinct_split/disitinct_split.groovy
+++ b/regression-test/suites/nereids_rules_p0/distinct_split/disitinct_split.groovy
@@ -207,4 +207,10 @@ suite("distinct_split") {
         group by grouping sets((a,b),(c));"""
         exception "The query contains multi count distinct or sum distinct, each can't have multi columns"
     }
-}
\ No newline at end of file
+
+    //----------------test null hash join ------------------------
+    sql "drop table if exists test_distinct_multi_null_hash;"
+    sql "create table test_distinct_multi_null_hash(a int, b int, c int, d varchar(10), e date) distributed by hash(a) properties('replication_num'='1');"
+    sql "insert into test_distinct_multi_null_hash values(1,null,null,null,'2024-12-08');"
+    qt_null_hash "SELECT a, b, count(distinct c,e), count(distinct concat(d,e))/count(distinct e) FROM test_distinct_multi_null_hash where e = '2024-12-08' GROUP BY a, b;"
+}

From 46109a77001ab0472a33910f826b72a3334c2f37 Mon Sep 17 00:00:00 2001
From: shee <13843187+qzsee@users.noreply.github.com>
Date: Fri, 10 Jan 2025 12:49:21 +0800
Subject: [PATCH 099/140] [fix](compaction) fix compaction producer hold for
 permits leak (#45664)

compaction producer hold
<img width="1083" alt="Clipboard_Screenshot_1734609076"
src="https://github.com/user-attachments/assets/0e675cf3-3235-47bc-ba07-97b1a90fa2c6"
/>

compaction threads is free
<img width="1044" alt="Clipboard_Screenshot_1734609095"
src="https://github.com/user-attachments/assets/147fccc1-b563-4160-9062-456bdca980c2"
/>
so judge permits has alread leak

Co-authored-by: garenshi <garenshi@tencent.com>
Co-authored-by: camby <cambyzhu@tencent.com>
---
 be/src/olap/olap_server.cpp | 36 ++++++++++++++++--------------------
 1 file changed, 16 insertions(+), 20 deletions(-)

diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index 90d0883984e78b6..2b77dc7e7a6f8cd 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -698,20 +698,6 @@ void StorageEngine::_compaction_tasks_producer_callback() {
                     last_base_score_update_time = cur_time;
                 }
             }
-            std::unique_ptr<ThreadPool>& thread_pool =
-                    (compaction_type == CompactionType::CUMULATIVE_COMPACTION)
-                            ? _cumu_compaction_thread_pool
-                            : _base_compaction_thread_pool;
-            VLOG_CRITICAL << "compaction thread pool. type: "
-                          << (compaction_type == CompactionType::CUMULATIVE_COMPACTION ? "CUMU"
-                                                                                       : "BASE")
-                          << ", num_threads: " << thread_pool->num_threads()
-                          << ", num_threads_pending_start: "
-                          << thread_pool->num_threads_pending_start()
-                          << ", num_active_threads: " << thread_pool->num_active_threads()
-                          << ", max_threads: " << thread_pool->max_threads()
-                          << ", min_threads: " << thread_pool->min_threads()
-                          << ", num_total_queued_tasks: " << thread_pool->get_queue_size();
             std::vector<TabletSharedPtr> tablets_compaction =
                     _generate_compaction_tasks(compaction_type, data_dirs, check_score);
             if (tablets_compaction.size() == 0) {
@@ -1066,23 +1052,33 @@ Status StorageEngine::_submit_compaction_task(TabletSharedPtr tablet,
                 (compaction_type == CompactionType::CUMULATIVE_COMPACTION)
                         ? _cumu_compaction_thread_pool
                         : _base_compaction_thread_pool;
+        VLOG_CRITICAL << "compaction thread pool. type: "
+                      << (compaction_type == CompactionType::CUMULATIVE_COMPACTION ? "CUMU"
+                                                                                   : "BASE")
+                      << ", num_threads: " << thread_pool->num_threads()
+                      << ", num_threads_pending_start: " << thread_pool->num_threads_pending_start()
+                      << ", num_active_threads: " << thread_pool->num_active_threads()
+                      << ", max_threads: " << thread_pool->max_threads()
+                      << ", min_threads: " << thread_pool->min_threads()
+                      << ", num_total_queued_tasks: " << thread_pool->get_queue_size();
         auto st = thread_pool->submit_func([tablet, compaction = std::move(compaction),
                                             compaction_type, permits, force, this]() {
+            Defer defer {[&]() {
+                if (!force) {
+                    _permit_limiter.release(permits);
+                }
+                _pop_tablet_from_submitted_compaction(tablet, compaction_type);
+                tablet->compaction_stage = CompactionStage::NOT_SCHEDULED;
+            }};
             if (!tablet->can_do_compaction(tablet->data_dir()->path_hash(), compaction_type)) {
                 LOG(INFO) << "Tablet state has been changed, no need to begin this compaction "
                              "task, tablet_id="
                           << tablet->tablet_id() << ", tablet_state=" << tablet->tablet_state();
-                _pop_tablet_from_submitted_compaction(tablet, compaction_type);
                 return;
             }
             tablet->compaction_stage = CompactionStage::EXECUTING;
             TEST_SYNC_POINT_RETURN_WITH_VOID("olap_server::execute_compaction");
             tablet->execute_compaction(*compaction);
-            if (!force) {
-                _permit_limiter.release(permits);
-            }
-            _pop_tablet_from_submitted_compaction(tablet, compaction_type);
-            tablet->compaction_stage = CompactionStage::NOT_SCHEDULED;
         });
         if (!st.ok()) {
             if (!force) {

From 4c8c502f75ada9789225bbf85181e8e7ca8ccc28 Mon Sep 17 00:00:00 2001
From: wuwenchi <wuwenchi@selectdb.com>
Date: Fri, 10 Jan 2025 14:09:55 +0800
Subject: [PATCH 100/140] [fix](iceberg)Different catalogs should use different
 client pools (#46694)

### What problem does this PR solve?

Problem Summary:

1. This cached client pool should belong to the catalog level, each
catalog has its own pool.
Otherwise, different dlf configurations will be connected to the same
dlf.
2. The catalog name should be used to initialize the catalog, not the
uid in the dlf.
---
 .../iceberg/HiveCompatibleCatalog.java        |  4 +-
 .../iceberg/IcebergDLFExternalCatalog.java    |  6 +--
 .../iceberg/IcebergGlueExternalCatalog.java   |  6 ---
 .../datasource/iceberg/dlf/DLFCatalog.java    |  2 +-
 .../dlf/client/DLFCachedClientPool.java       | 19 ++++-----
 .../client/IcebergDLFExternalCatalogTest.java | 41 +++++++++++++++++++
 6 files changed, 53 insertions(+), 25 deletions(-)
 create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/dlf/client/IcebergDLFExternalCatalogTest.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/HiveCompatibleCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/HiveCompatibleCatalog.java
index 6431b02308b7a21..49123d2b8f463c2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/HiveCompatibleCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/HiveCompatibleCatalog.java
@@ -45,11 +45,11 @@ public abstract class HiveCompatibleCatalog extends BaseMetastoreCatalog impleme
     protected Configuration conf;
     protected ClientPool<IMetaStoreClient, TException> clients;
     protected FileIO fileIO;
-    protected String uid;
+    protected String catalogName;
 
     public void initialize(String name, FileIO fileIO,
                            ClientPool<IMetaStoreClient, TException> clients) {
-        this.uid = name;
+        this.catalogName = name;
         this.fileIO = fileIO;
         this.clients = clients;
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergDLFExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergDLFExternalCatalog.java
index e4d8b2f55c43f1f..d3f192754ab4db5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergDLFExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergDLFExternalCatalog.java
@@ -22,8 +22,6 @@
 import org.apache.doris.datasource.property.PropertyConverter;
 import org.apache.doris.datasource.property.constants.HMSProperties;
 
-import com.aliyun.datalake.metastore.common.DataLakeConfig;
-
 import java.util.Map;
 
 public class IcebergDLFExternalCatalog extends IcebergExternalCatalog {
@@ -43,8 +41,8 @@ protected void initCatalog() {
         dlfCatalog.setConf(getConfiguration());
         // initialize catalog
         Map<String, String> catalogProperties = catalogProperty.getHadoopProperties();
-        String dlfUid = catalogProperties.get(DataLakeConfig.CATALOG_USER_ID);
-        dlfCatalog.initialize(dlfUid, catalogProperties);
+        String catalogName = getName();
+        dlfCatalog.initialize(catalogName, catalogProperties);
         catalog = dlfCatalog;
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java
index ffe48e68a49af20..f9f602033c29a0a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java
@@ -27,7 +27,6 @@
 import org.apache.iceberg.aws.glue.GlueCatalog;
 import org.apache.iceberg.aws.s3.S3FileIOProperties;
 
-import java.util.List;
 import java.util.Map;
 
 public class IcebergGlueExternalCatalog extends IcebergExternalCatalog {
@@ -61,9 +60,4 @@ protected void initCatalog() {
         glueCatalog.initialize(getName(), catalogProperties);
         catalog = glueCatalog;
     }
-
-    @Override
-    protected List<String> listDatabaseNames() {
-        return metadataOps.listDatabaseNames();
-    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/dlf/DLFCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/dlf/DLFCatalog.java
index b9ffc006c61b0f2..c47ff7248d1a561 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/dlf/DLFCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/dlf/DLFCatalog.java
@@ -46,7 +46,7 @@ public void initialize(String name, Map<String, String> properties) {
     protected TableOperations newTableOps(TableIdentifier tableIdentifier) {
         String dbName = tableIdentifier.namespace().level(0);
         String tableName = tableIdentifier.name();
-        return new DLFTableOperations(this.conf, this.clients, this.fileIO, this.uid, dbName, tableName);
+        return new DLFTableOperations(this.conf, this.clients, this.fileIO, this.catalogName, dbName, tableName);
     }
 
     protected FileIO initializeFileIO(Map<String, String> properties, Configuration hadoopConf) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/dlf/client/DLFCachedClientPool.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/dlf/client/DLFCachedClientPool.java
index 23b814c13b8085e..9de0981e9809ce4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/dlf/client/DLFCachedClientPool.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/dlf/client/DLFCachedClientPool.java
@@ -31,13 +31,14 @@
 
 public class DLFCachedClientPool implements ClientPool<IMetaStoreClient, TException> {
 
-    private static volatile Cache<String, DLFClientPool> clientPoolCache;
-    private static final Object clientPoolCacheLock = new Object();
+    private Cache<String, DLFClientPool> clientPoolCache;
     private final Configuration conf;
     private final String endpoint;
     private final int clientPoolSize;
     private final long evictionInterval;
 
+    // This cached client pool should belong to the catalog level,
+    // each catalog has its own pool
     public DLFCachedClientPool(Configuration conf, Map<String, String> properties) {
         this.conf = conf;
         this.endpoint = conf.get("", "");
@@ -63,16 +64,10 @@ private long getEvictionInterval(Map<String, String> properties) {
     }
 
     private void initializeClientPoolCache() {
-        if (clientPoolCache == null) {
-            synchronized (clientPoolCacheLock) {
-                if (clientPoolCache == null) {
-                    clientPoolCache = Caffeine.newBuilder()
-                            .expireAfterAccess(evictionInterval, TimeUnit.MILLISECONDS)
-                            .removalListener((key, value, cause) -> ((DLFClientPool) value).close())
-                            .build();
-                }
-            }
-        }
+        clientPoolCache = Caffeine.newBuilder()
+                .expireAfterAccess(evictionInterval, TimeUnit.MILLISECONDS)
+                .removalListener((key, value, cause) -> ((DLFClientPool) value).close())
+                .build();
     }
 
     protected DLFClientPool clientPool() {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/dlf/client/IcebergDLFExternalCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/dlf/client/IcebergDLFExternalCatalogTest.java
new file mode 100644
index 000000000000000..bbd39b7b71bfc01
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/dlf/client/IcebergDLFExternalCatalogTest.java
@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.iceberg.dlf.client;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+
+public class IcebergDLFExternalCatalogTest {
+    @Test
+    public void testDatabaseList() {
+        HashMap<String, String> props = new HashMap<>();
+        Configuration conf = new Configuration();
+
+        DLFCachedClientPool cachedClientPool1 = new DLFCachedClientPool(conf, props);
+        DLFCachedClientPool cachedClientPool2 = new DLFCachedClientPool(conf, props);
+        DLFClientPool dlfClientPool1 = cachedClientPool1.clientPool();
+        DLFClientPool dlfClientPool2 = cachedClientPool2.clientPool();
+        // This cache should belong to the catalog level,
+        // so the object addresses of clients in different pools must be different
+        Assert.assertNotSame(dlfClientPool1, dlfClientPool2);
+
+    }
+}

From afadd682482c0fa84696e65ad049d00c827dee8e Mon Sep 17 00:00:00 2001
From: Petrichor <vinleexiao@163.com>
Date: Fri, 10 Jan 2025 15:00:30 +0800
Subject: [PATCH 101/140] [feat](nereids) implement showConvertLscCommand in
 nereids (#45631)

Issue Number: close https://github.com/apache/doris/issues/42781
---
 .../org/apache/doris/nereids/DorisParser.g4   |   2 +-
 .../nereids/parser/LogicalPlanBuilder.java    |  17 +++
 .../doris/nereids/trees/plans/PlanType.java   |   1 +
 .../plans/commands/ShowConvertLSCCommand.java | 116 ++++++++++++++++++
 .../trees/plans/visitor/CommandVisitor.java   |   5 +
 .../show_light_schema_change_process_nereids  |  24 ++++
 6 files changed, 164 insertions(+), 1 deletion(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowConvertLSCCommand.java
 create mode 100644 regression-test/suites/nereids_p0/ddl/show/show_light_schema_change_process_nereids

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index a1a3ac6d297754e..5fd21da747e7dd0 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -303,6 +303,7 @@ supportedShowStatement
         (LIKE STRING_LITERAL)?                                                      #showTableCreation
     | SHOW TABLET STORAGE FORMAT VERBOSE?                                           #showTabletStorageFormat
     | SHOW QUERY PROFILE queryIdPath=STRING_LITERAL? limitClause?                    #showQueryProfile
+    | SHOW CONVERT_LSC ((FROM | IN) database=multipartIdentifier)?                  #showConvertLsc
     ;
 
 supportedLoadStatement
@@ -391,7 +392,6 @@ unsupportedShowStatement
     | SHOW BUILD INDEX ((FROM | IN) database=multipartIdentifier)?
         wildWhere? sortClause? limitClause?                                         #showBuildIndex
     | SHOW (CLUSTERS | (COMPUTE GROUPS))                                            #showClusters
-    | SHOW CONVERT_LSC ((FROM | IN) database=multipartIdentifier)?                  #showConvertLsc
     | SHOW REPLICA STATUS FROM baseTableRef wildWhere?                              #showReplicaStatus
     | SHOW COPY ((FROM | IN) database=multipartIdentifier)?
         whereClause? sortClause? limitClause?                                       #showCopy
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 32f32f4f6c598c1..0432e06c3354722 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -277,6 +277,7 @@
 import org.apache.doris.nereids.DorisParser.ShowCollationContext;
 import org.apache.doris.nereids.DorisParser.ShowConfigContext;
 import org.apache.doris.nereids.DorisParser.ShowConstraintContext;
+import org.apache.doris.nereids.DorisParser.ShowConvertLscContext;
 import org.apache.doris.nereids.DorisParser.ShowCreateCatalogContext;
 import org.apache.doris.nereids.DorisParser.ShowCreateDatabaseContext;
 import org.apache.doris.nereids.DorisParser.ShowCreateMTMVContext;
@@ -564,6 +565,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowCollationCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowConfigCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowConstraintsCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowConvertLSCCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateCatalogCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateDatabaseCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateMTMVCommand;
@@ -5436,5 +5438,20 @@ public LogicalPlan visitUseDatabase(UseDatabaseContext ctx) {
         return ctx.catalog != null ? new UseCommand(ctx.catalog.getText(), ctx.database.getText())
                 : new UseCommand(ctx.database.getText());
     }
+
+    @Override
+    public LogicalPlan visitShowConvertLsc(ShowConvertLscContext ctx) {
+        if (ctx.database == null) {
+            return new ShowConvertLSCCommand(null);
+        }
+        List<String> parts = visitMultipartIdentifier(ctx.database);
+        String databaseName = parts.get(parts.size() - 1);
+        if (parts.size() == 2 && !InternalCatalog.INTERNAL_CATALOG_NAME.equalsIgnoreCase(parts.get(0))) {
+            throw new ParseException("The execution of this command is restricted to the internal catalog only.");
+        } else if (parts.size() > 2) {
+            throw new ParseException("Only one dot can be in the name: " + String.join(".", parts));
+        }
+        return new ShowConvertLSCCommand(databaseName);
+    }
 }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 5120bbbf4b2dd38..1c0cb26f0f07024 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -262,6 +262,7 @@ public enum PlanType {
     SHOW_WARNING_ERROR_COUNT_COMMAND,
     SHOW_WHITE_LIST_COMMAND,
     SHOW_TABLETS_BELONG_COMMAND,
+    SHOW_CONVERT_LSC_COMMAND,
     SYNC_COMMAND,
     RECOVER_DATABASE_COMMAND,
     RECOVER_TABLE_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowConvertLSCCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowConvertLSCCommand.java
new file mode 100644
index 000000000000000..705e9efd7f04b9a
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowConvertLSCCommand.java
@@ -0,0 +1,116 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.analysis.RedirectStatus;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.ColumnIdFlushDaemon;
+import org.apache.doris.catalog.ColumnIdFlushDaemon.FlushStatus;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.ShowResultSetMetaData;
+import org.apache.doris.qe.StmtExecutor;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Represents the command for SHOW CONVERT LIGHT SCHEMA CHANGE PROCESS .
+ */
+public class ShowConvertLSCCommand extends ShowCommand {
+    private static final int COLUMN_LENGTH = 30;
+    private final String dbName;
+
+    public ShowConvertLSCCommand(String dbName) {
+        super(PlanType.SHOW_CONVERT_LSC_COMMAND);
+        this.dbName = dbName;
+    }
+
+    @Override
+    public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        // Check if the user has ADMIN or OPERATOR privileges
+        if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)
+                && !Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(),
+                PrivPredicate.OPERATOR)) {
+            ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN/OPERATOR");
+        }
+        return handleShowConvertLSC();
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitShowConvertLscCommand(this, context);
+    }
+
+    /**
+     * get show result set metadata.
+     */
+    public ShowResultSetMetaData getMetaData() {
+        ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
+        Column databaseColumn = new Column("database", ScalarType.createVarcharType(COLUMN_LENGTH));
+        Column tableNameColumn = new Column("table", ScalarType.createVarcharType(COLUMN_LENGTH));
+        Column statusColumn = new Column("status", ScalarType.createVarcharType(COLUMN_LENGTH));
+        builder.addColumn(databaseColumn);
+        builder.addColumn(tableNameColumn);
+        builder.addColumn(statusColumn);
+        return builder.build();
+    }
+
+    private ShowResultSet handleShowConvertLSC() {
+        ColumnIdFlushDaemon columnIdFlusher = Env.getCurrentEnv().getColumnIdFlusher();
+        columnIdFlusher.readLock();
+        List<List<String>> rows;
+        try {
+            Map<String, Map<String, FlushStatus>> resultCollector =
+                    columnIdFlusher.getResultCollector();
+            rows = new ArrayList<>();
+            if (dbName != null) {
+                Map<String, ColumnIdFlushDaemon.FlushStatus> tblNameToStatus = resultCollector.get(dbName);
+                if (tblNameToStatus != null) {
+                    tblNameToStatus.forEach((tblName, status) -> {
+                        List<String> row = Arrays.asList(dbName, tblName, status.getMsg());
+                        rows.add(row);
+                    });
+                }
+            } else {
+                resultCollector.forEach((dbName, tblNameToStatus) ->
+                        tblNameToStatus.forEach((tblName, status) -> {
+                            List<String> row = Arrays.asList(dbName, tblName, status.getMsg());
+                            rows.add(row);
+                        }));
+            }
+        } finally {
+            columnIdFlusher.readUnlock();
+        }
+        return new ShowResultSet(getMetaData(), rows);
+    }
+
+    @Override
+    public RedirectStatus toRedirectStatus() {
+        return RedirectStatus.FORWARD_NO_SYNC;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 15adde32ffb3eed..0f5cafa919227c8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -98,6 +98,7 @@
 import org.apache.doris.nereids.trees.plans.commands.ShowCollationCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowConfigCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowConstraintsCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowConvertLSCCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateCatalogCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateDatabaseCommand;
 import org.apache.doris.nereids.trees.plans.commands.ShowCreateMTMVCommand;
@@ -741,6 +742,10 @@ default R visitShowQueryProfileCommand(ShowQueryProfileCommand showQueryProfileC
         return visitCommand(showQueryProfileCommand, context);
     }
 
+    default R visitShowConvertLscCommand(ShowConvertLSCCommand showConvertLSCCommand, C context) {
+        return visitCommand(showConvertLSCCommand, context);
+    }
+
     default R visitSwitchCommand(SwitchCommand switchCommand, C context) {
         return visitCommand(switchCommand, context);
     }
diff --git a/regression-test/suites/nereids_p0/ddl/show/show_light_schema_change_process_nereids b/regression-test/suites/nereids_p0/ddl/show/show_light_schema_change_process_nereids
new file mode 100644
index 000000000000000..50041209d202ee0
--- /dev/null
+++ b/regression-test/suites/nereids_p0/ddl/show/show_light_schema_change_process_nereids
@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("show_light_schema_change_process_nereids") {
+   sql """drop database if exists testdb;""
+   sql """create database testdb;"""
+   checkNereidsExecute("""SHOW CONVERT_LIGHT_SCHEMA_CHANGE_PROCESS FROM testdb;""")
+   checkNereidsExecute("""SHOW CONVERT_LIGHT_SCHEMA_CHANGE_PROCESS;""")
+   sql """drop database if exists testdb;""
+}

From 78f0dbe7efd11cb02677ef91dc04699a25361bfa Mon Sep 17 00:00:00 2001
From: deardeng <dengxin@selectdb.com>
Date: Fri, 10 Jan 2025 15:11:14 +0800
Subject: [PATCH 102/140] [fix](test) Fix test frontend run in cloud (#46760)

### What problem does this PR solve?

Issue Number: close #xxx

Related PR: #xxx

Problem Summary:

Due to the 5-minute limit for dropping FE on the cloud, the drop FE
process will not run in this case. There are dedicated Docker cases on
the cloud to handle such scenarios.
---
 .../suites/node_p0/test_frontend.groovy         | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/regression-test/suites/node_p0/test_frontend.groovy b/regression-test/suites/node_p0/test_frontend.groovy
index 99d6068f7ba945d..29ee3e06315a303 100644
--- a/regression-test/suites/node_p0/test_frontend.groovy
+++ b/regression-test/suites/node_p0/test_frontend.groovy
@@ -16,6 +16,17 @@
 // under the License.
 
 suite("test_frontend", "nonConcurrent") {
+    def res = sql """SHOW FRONTENDS DISKS"""
+    assertTrue(res.size() != 0)
+
+    def res2 = sql """SHOW FRONTENDS Disks"""
+    assertTrue(res2.size() != 0)
+
+    if (Config.isCloudMode()) {
+        // In the test_sql_mode_node_mgr regression case, there is already a similar and more complex case. This case is redundant. Additionally, there is a 5-minute limit for dropping FE on the cloud.
+        // so ignore it in cloud
+        return;
+    }
     def address = "127.0.0.1"
     def notExistPort = 12345
 
@@ -43,10 +54,4 @@ suite("test_frontend", "nonConcurrent") {
         result = sql """SHOW FRONTENDS;"""
         logger.debug("result:${result}")
     }
-
-    def res = sql """SHOW FRONTENDS DISKS"""
-    assertTrue(res.size() != 0)
-
-    def res2 = sql """SHOW FRONTENDS Disks"""
-    assertTrue(res2.size() != 0)
 }

From da3d2aff7cd511c59f0eabfb3dfee9b3960eec92 Mon Sep 17 00:00:00 2001
From: Jerry Hu <hushenggang@selectdb.com>
Date: Fri, 10 Jan 2025 16:44:42 +0800
Subject: [PATCH 103/140] [fix](pipelien) should not finalize probe when wake
 up early in SetProbeSinkOperatorX (#46706)

### What problem does this PR solve?

```
*** Query id: 80819fcc223e4a45-b46155de6e0c4eee ***
*** is nereids: 1 ***
*** tablet id: 0 ***
*** Aborted at 1736352810 (unix time) try "date -d @1736352810" if you are using GNU date ***
*** Current BE git commitID: 08683cbaf5 ***
*** SIGSEGV address not mapped to object (@0x38) received by PID 8736 (TID 11549 OR 0x7f8dd0922640) from PID 56; stack trace: ***
 0# doris::signal::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*) at /root/doris_branch-3.0/doris/be/src/common/signal_handler.h:421
 1# PosixSignals::chained_handler(int, siginfo*, void*) [clone .part.0] in /usr/lib/jvm/java-17-openjdk-amd64/lib/server/libjvm.so
 2# JVM_handle_linux_signal in /usr/lib/jvm/java-17-openjdk-amd64/lib/server/libjvm.so
 3# 0x00007F92019CA520 in /lib/x86_64-linux-gnu/libc.so.6
 4# auto doris::pipeline::SetProbeSinkOperatorX::_refresh_hash_table(doris::pipeline::SetProbeSinkLocalState&)::{lambda(auto:1&&)#1}::operator(), HashTableNoState>, DefaultHash, HashTableGrower<10ul>, Allocator > >&>(doris::vectorized::MethodSerialized, HashTableNoState>, DefaultHash, HashTableGrower<10ul>, Allocator > >&) const at /root/doris_branch-3.0/doris/be/src/pipeline/exec/set_probe_sink_operator.cpp:213
 5# doris::pipeline::SetProbeSinkOperatorX::_finalize_probe(doris::pipeline::SetProbeSinkLocalState&) at /root/doris_branch-3.0/doris/be/src/pipeline/exec/set_probe_sink_operator.cpp:184
 6# doris::pipeline::SetProbeSinkOperatorX::sink(doris::RuntimeState*, doris::vectorized::Block*, bool) at /root/doris_branch-3.0/doris/be/src/pipeline/exec/set_probe_sink_operator.cpp:98
 7# doris::pipeline::PipelineTask::execute(bool*) at /root/doris_branch-3.0/doris/be/src/pipeline/pipeline_task.cpp:387
 8# doris::pipeline::TaskScheduler::_do_work(unsigned long) at /root/doris_branch-3.0/doris/be/src/pipeline/task_scheduler.cpp:138
 9# doris::ThreadPool::dispatch_thread() in /mnt/ssd01/doris-branch40preview/NEREIDS_ASAN/be/lib/doris_be
10# doris::Thread::supervise_thread(void*) at /root/doris_branch-3.0/doris/be/src/util/thread.cpp:499
11# start_thread at ./nptl/pthread_create.c:442
12# 0x00007F9201AAE850 at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:83
```
---
 be/src/pipeline/exec/set_probe_sink_operator.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/be/src/pipeline/exec/set_probe_sink_operator.cpp b/be/src/pipeline/exec/set_probe_sink_operator.cpp
index 929062ec80b6c5e..a758c2ae3c60023 100644
--- a/be/src/pipeline/exec/set_probe_sink_operator.cpp
+++ b/be/src/pipeline/exec/set_probe_sink_operator.cpp
@@ -22,6 +22,7 @@
 #include <memory>
 
 #include "pipeline/exec/operator.h"
+#include "pipeline/pipeline_task.h"
 #include "vec/common/hash_table/hash_table_set_probe.h"
 
 namespace doris {
@@ -70,7 +71,7 @@ Status SetProbeSinkOperatorX<is_intersect>::sink(RuntimeState* state, vectorized
     SCOPED_TIMER(local_state.exec_time_counter());
     COUNTER_UPDATE(local_state.rows_input_counter(), (int64_t)in_block->rows());
 
-    uint32_t probe_rows = cast_set<uint32_t>(in_block->rows());
+    const auto probe_rows = cast_set<uint32_t>(in_block->rows());
     if (probe_rows > 0) {
         {
             SCOPED_TIMER(local_state._extract_probe_data_timer);
@@ -93,7 +94,7 @@ Status SetProbeSinkOperatorX<is_intersect>::sink(RuntimeState* state, vectorized
                 local_state._shared_state->hash_table_variants->method_variant));
     }
 
-    if (eos) {
+    if (eos && !state->get_task()->wake_up_early()) {
         _finalize_probe(local_state);
     }
     return Status::OK();

From 3f082e98abc3bf54ee74baf922f596a971dd8eed Mon Sep 17 00:00:00 2001
From: walter <maochuan@selectdb.com>
Date: Fri, 10 Jan 2025 17:01:33 +0800
Subject: [PATCH 104/140] [fix](suite) Fix qt_master_sql arg type (#46772)

---
 .../main/groovy/org/apache/doris/regression/suite/Suite.groovy  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index 3995371db952da6..5887fd607d44a91 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -1292,7 +1292,7 @@ class Suite implements GroovyInterceptable {
                 } else if (tag.contains("target_sql")) {
                     tupleResult = JdbcUtils.executeToStringList(context.getTargetConnection(this), (String) arg)
                 } else if (tag.contains("master_sql")) {
-                    tupleResult = JdbcUtils.executeToStringList(context.getMasterConnection(), (PreparedStatement) arg)
+                    tupleResult = JdbcUtils.executeToStringList(context.getMasterConnection(), (String) arg)
                 } else {
                     tupleResult = JdbcUtils.executeToStringList(context.getConnection(),  (String) arg)
                 }

From 6422af2e4a8458a09dded1e280619134529acfea Mon Sep 17 00:00:00 2001
From: abmdocrt <lianyukang@selectdb.com>
Date: Fri, 10 Jan 2025 17:05:21 +0800
Subject: [PATCH 105/140] [Fix](test) Fix infinate loop in
 test_cumu_compaction_with_delete case (#46719)

---
 .../compaction/test_cumu_compaction_with_delete.groovy | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/regression-test/suites/compaction/test_cumu_compaction_with_delete.groovy b/regression-test/suites/compaction/test_cumu_compaction_with_delete.groovy
index 7ad5adf069ed7f0..4ac3953f55a0f1b 100644
--- a/regression-test/suites/compaction/test_cumu_compaction_with_delete.groovy
+++ b/regression-test/suites/compaction/test_cumu_compaction_with_delete.groovy
@@ -69,6 +69,11 @@ suite("test_cumu_compaction_with_delete") {
                 break;
             }
             Thread.sleep(10000)
+
+            def duration = System.currentTimeMillis() - now
+            if(duration > 10 * 60 * 1000) {
+                assertTrue(false)
+            }
         }
         def time_diff = System.currentTimeMillis() - now
         logger.info("time_diff:" + time_diff)
@@ -108,6 +113,11 @@ suite("test_cumu_compaction_with_delete") {
                 break;
             }
             Thread.sleep(10000)
+
+            def duration = System.currentTimeMillis() - now
+            if(duration > 10 * 60 * 1000) {
+                assertTrue(false)
+            }
         }
         def time_diff = System.currentTimeMillis() - now
         logger.info("time_diff:" + time_diff)

From 358f9175c9438566b687c1da1d09944b963efade Mon Sep 17 00:00:00 2001
From: abmdocrt <lianyukang@selectdb.com>
Date: Fri, 10 Jan 2025 17:07:00 +0800
Subject: [PATCH 106/140] [Fix](test) Fix global variables in full compaction
 test (#46747)


From 35fc172ba676a68213f14a4ae03212da7fec5c87 Mon Sep 17 00:00:00 2001
From: zgxme <zhenggaoxiong@selectdb.com>
Date: Fri, 10 Jan 2025 18:01:04 +0800
Subject: [PATCH 107/140] [Opt](docker) kerberos docker healthy check (#46662)

### What problem does this PR solve?
Previously, kerberos docker did not have health checks which could lead
to unstable tests.
---
 .../kerberos/entrypoint-hive-master-2.sh      | 20 ++++++++++++++++---
 .../kerberos/entrypoint-hive-master.sh        | 20 ++++++++++++++++---
 .../health-checks/hadoop-health-check.sh      |  1 +
 .../kerberos/health-checks/health.sh          |  1 +
 .../health-checks/hive-health-check-2.sh      | 20 +++++++++++++++++++
 .../health-checks/hive-health-check.sh        | 20 +++++++++++++++++++
 .../docker-compose/kerberos/kerberos.yaml.tpl | 16 +++++++++++----
 .../thirdparties/run-thirdparties-docker.sh   | 14 +++++--------
 8 files changed, 93 insertions(+), 19 deletions(-)
 mode change 100644 => 100755 docker/thirdparties/docker-compose/kerberos/health-checks/health.sh
 create mode 100755 docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check-2.sh
 create mode 100755 docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check.sh

diff --git a/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master-2.sh b/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master-2.sh
index c21460c3a57a0f5..eb95c5cb697619f 100755
--- a/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master-2.sh
+++ b/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master-2.sh
@@ -25,12 +25,26 @@ cp /etc/trino/conf/presto-server.keytab /keytabs/other-presto-server.keytab
 cp /keytabs/update-location.sh /etc/hadoop-init.d/update-location.sh
 /usr/local/hadoop-run.sh &
 
-sleep 30
+# check healthy hear
+echo "Waiting for hadoop to be healthy"
+
+for i in {1..10}; do
+    if /usr/local/health.sh; then
+        echo "Hadoop is healthy"
+        break
+    fi
+    echo "Hadoop is not healthy yet. Retrying in 20 seconds..."
+    sleep 20
+done
+
+if [ $i -eq 10 ]; then
+    echo "Hadoop did not become healthy after 120 attempts. Exiting."
+    exit 1
+fi
 
 echo "Init kerberos test data"
 kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master-2@OTHERREALM.COM
 hive  -f /usr/local/sql/create_kerberos_hive_table.sql
-
-sleep 20
+touch /mnt/SUCCESS
 
 tail -f /dev/null
diff --git a/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh b/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh
index 62924992219a1d6..76f49724297a619 100755
--- a/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh
+++ b/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh
@@ -23,12 +23,26 @@ mkdir -p /etc/hadoop-init.d/
 cp /etc/trino/conf/* /keytabs/
 /usr/local/hadoop-run.sh &
 
-sleep 30
+# check healthy hear
+echo "Waiting for hadoop to be healthy"
+
+for i in {1..10}; do
+    if /usr/local/health.sh; then
+        echo "Hadoop is healthy"
+        break
+    fi
+    echo "Hadoop is not healthy yet. Retrying in 20 seconds..."
+    sleep 20
+done
+
+if [ $i -eq 10 ]; then
+    echo "Hadoop did not become healthy after 120 attempts. Exiting."
+    exit 1
+fi
 
 echo "Init kerberos test data"
 kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master@LABS.TERADATA.COM
 hive  -f /usr/local/sql/create_kerberos_hive_table.sql
-
-sleep 20
+touch /mnt/SUCCESS
 
 tail -f /dev/null
diff --git a/docker/thirdparties/docker-compose/kerberos/health-checks/hadoop-health-check.sh b/docker/thirdparties/docker-compose/kerberos/health-checks/hadoop-health-check.sh
index 190fa838d6f64d1..77df431d85ac3ba 100755
--- a/docker/thirdparties/docker-compose/kerberos/health-checks/hadoop-health-check.sh
+++ b/docker/thirdparties/docker-compose/kerberos/health-checks/hadoop-health-check.sh
@@ -32,6 +32,7 @@ fi
 FAILED=$(supervisorctl status | grep -v RUNNING || true)
 
 if [ "$FAILED" == "" ]; then
+  echo "All services are running"
   exit 0
 else
   echo "Some of the services are failing: ${FAILED}"
diff --git a/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh b/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh
old mode 100644
new mode 100755
index 515f37e36ac9e37..473d7ceaeb6166b
--- a/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh
+++ b/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh
@@ -32,3 +32,4 @@ if test -d "${HEALTH_D}"; then
         "${health_script}" &>> /var/log/container-health.log || exit 1
     done
 fi
+exit 0
diff --git a/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check-2.sh b/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check-2.sh
new file mode 100755
index 000000000000000..854524dac1fcff4
--- /dev/null
+++ b/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check-2.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master-2@OTHERREALM.COM
+beeline -u "jdbc:hive2://localhost:10000/default;principal=hive/hadoop-master-2@OTHERREALM.COM" -e "show databases;"
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check.sh b/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check.sh
new file mode 100755
index 000000000000000..4d3d86f69a25c4a
--- /dev/null
+++ b/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master@LABS.TERADATA.COM
+beeline -u "jdbc:hive2://localhost:10000/default;principal=hive/hadoop-master@LABS.TERADATA.COM" -e "show databases;"
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl b/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl
index 6aa353f3e0cfc82..e635ed6bb27f344 100644
--- a/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl
+++ b/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl
@@ -24,13 +24,17 @@ services:
       - ./sql:/usr/local/sql
       - ./common/hadoop/apply-config-overrides.sh:/etc/hadoop-init.d/00-apply-config-overrides.sh
       - ./common/hadoop/hadoop-run.sh:/usr/local/hadoop-run.sh
+      - ./health-checks/health.sh:/usr/local/health.sh
       - ./health-checks/hadoop-health-check.sh:/etc/health.d/hadoop-health-check.sh
+      - ./health-checks/hive-health-check.sh:/etc/health.d/hive-health-check.sh
       - ./entrypoint-hive-master.sh:/usr/local/entrypoint-hive-master.sh
-    restart: on-failure
     hostname: hadoop-master
     entrypoint: /usr/local/entrypoint-hive-master.sh
     healthcheck:
-      test: ./health-checks/health.sh
+      test: ["CMD", "ls", "/mnt/SUCCESS"]
+      interval: 20s
+      timeout: 60s
+      retries: 120
     ports:
       - "5806:5006"
       - "8820:8020"
@@ -46,17 +50,21 @@ services:
     image: doristhirdpartydocker/trinodb:hdp3.1-hive-kerberized-2_96
     container_name: doris--kerberos2
     hostname: hadoop-master-2
-    restart: on-failure
     volumes:
       - ./two-kerberos-hives:/keytabs
       - ./sql:/usr/local/sql
       - ./common/hadoop/apply-config-overrides.sh:/etc/hadoop-init.d/00-apply-config-overrides.sh
       - ./common/hadoop/hadoop-run.sh:/usr/local/hadoop-run.sh
+      - ./health-checks/health.sh:/usr/local/health.sh
       - ./health-checks/hadoop-health-check.sh:/etc/health.d/hadoop-health-check.sh
+      - ./health-checks/hive-health-check-2.sh:/etc/health.d/hive-health-check-2.sh
       - ./entrypoint-hive-master-2.sh:/usr/local/entrypoint-hive-master-2.sh
     entrypoint: /usr/local/entrypoint-hive-master-2.sh
     healthcheck:
-      test: ./health-checks/health.sh
+      test: ["CMD", "ls", "/mnt/SUCCESS"]
+      interval: 20s
+      timeout: 60s
+      retries: 120
     ports:
       - "15806:5006"
       - "18820:8020"
diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh
index fd9558eef558b06..e3769025fec62d9 100755
--- a/docker/thirdparties/run-thirdparties-docker.sh
+++ b/docker/thirdparties/run-thirdparties-docker.sh
@@ -708,6 +708,11 @@ if [[ "${RUN_MINIO}" -eq 1 ]]; then
     pids["minio"]=$!
 fi
 
+if [[ "${RUN_KERBEROS}" -eq 1 ]]; then
+    start_kerberos > start_kerberos.log 2>&1 &
+    pids["kerberos"]=$!
+fi
+
 echo "waiting all dockers starting done"
 
 for compose in "${!pids[@]}"; do
@@ -727,15 +732,6 @@ for compose in "${!pids[@]}"; do
     fi
 done
 
-if [[ "${RUN_KERBEROS}" -eq 1 ]]; then
-    echo "Starting Kerberos after all other components..."
-    start_kerberos > start_kerberos.log 2>&1
-    if [ $? -ne 0 ]; then
-        echo "Kerberos startup failed"
-        cat start_kerberos.log
-        exit 1
-    fi
-fi
 echo "docker started"
 docker ps -a --format "{{.ID}} | {{.Image}} | {{.Status}}"
 echo "all dockers started successfully"

From a7df843bfdaf02bb4b8c110dc00058f7e480b43f Mon Sep 17 00:00:00 2001
From: daidai <changyuwei@selectdb.com>
Date: Fri, 10 Jan 2025 18:18:27 +0800
Subject: [PATCH 108/140] [fix](hive)fix select count(*) hive full acid tb opt
 error. (#46732)

### What problem does this PR solve?
Problem Summary:
before pr : #44038
In the previous PR, the generation method of split in the count( * )
scenario was optimized.
However, there were some problems with the hive acid table. This PR
mainly fixes this and adds tests.
In the count( * ) scenario, reading the hive full acid table cannot be
optimized, and the file still needs to be split (merge on read is
required), and the hive insert only acid table does not need to be
split.
---
 .../format/table/transactional_hive_reader.cpp    |  1 +
 .../datasource/hive/source/HiveScanNode.java      |  4 ++--
 .../hive/test_transactional_hive.out              | 15 +++++++++++++++
 .../hive/test_hive_translation_insert_only.out    |  9 +++++++++
 .../hive/test_transactional_hive.groovy           | 12 ++++++++++++
 .../hive/test_hive_translation_insert_only.groovy |  5 +++++
 6 files changed, 44 insertions(+), 2 deletions(-)

diff --git a/be/src/vec/exec/format/table/transactional_hive_reader.cpp b/be/src/vec/exec/format/table/transactional_hive_reader.cpp
index 18642ab1218b4d2..caf242700180c5a 100644
--- a/be/src/vec/exec/format/table/transactional_hive_reader.cpp
+++ b/be/src/vec/exec/format/table/transactional_hive_reader.cpp
@@ -205,6 +205,7 @@ Status TransactionalHiveReader::init_row_filters(const TFileRangeDesc& range,
         ++num_delete_files;
     }
     if (num_delete_rows > 0) {
+        orc_reader->set_push_down_agg_type(TPushAggOp::NONE);
         orc_reader->set_delete_rows(&_delete_rows);
         COUNTER_UPDATE(_transactional_orc_profile.num_delete_files, num_delete_files);
         COUNTER_UPDATE(_transactional_orc_profile.num_delete_rows, num_delete_rows);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
index c559570432fb17d..08cf65824472533 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
@@ -289,12 +289,12 @@ private void getFileSplitByPartitions(HiveMetaStoreCache cache, List<HivePartiti
          * we don't need to split the file because for parquet/orc format, only metadata is read.
          * If we split the file, we will read metadata of a file multiple times, which is not efficient.
          *
-         * - Hive Transactional Table may need merge on read, so do not apply this optimization.
+         * - Hive Full Acid Transactional Table may need merge on read, so do not apply this optimization.
          * - If the file format is not parquet/orc, eg, text, we need to split the file to increase the parallelism.
          */
         boolean needSplit = true;
         if (getPushDownAggNoGroupingOp() == TPushAggOp.COUNT
-                && hiveTransaction != null) {
+                && !(hmsTable.isHiveTransactionalTable() && hmsTable.isFullAcidTable())) {
             int totalFileNum = 0;
             for (FileCacheValue fileCacheValue : fileCaches) {
                 if (fileCacheValue.getFiles() != null) {
diff --git a/regression-test/data/external_table_p0/hive/test_transactional_hive.out b/regression-test/data/external_table_p0/hive/test_transactional_hive.out
index 060fa8c048e5a05..94e32a43db7f3bb 100644
--- a/regression-test/data/external_table_p0/hive/test_transactional_hive.out
+++ b/regression-test/data/external_table_p0/hive/test_transactional_hive.out
@@ -122,3 +122,18 @@ F
 -- !16 --
 4	DD
 
+-- !count_1 --
+3
+
+-- !count_2 --
+6
+
+-- !count_3 --
+4
+
+-- !count_4 --
+3
+
+-- !count_5 --
+3
+
diff --git a/regression-test/data/external_table_p2/hive/test_hive_translation_insert_only.out b/regression-test/data/external_table_p2/hive/test_hive_translation_insert_only.out
index e4bdb3fe32d44bc..f43a630f4a387c1 100644
--- a/regression-test/data/external_table_p2/hive/test_hive_translation_insert_only.out
+++ b/regression-test/data/external_table_p2/hive/test_hive_translation_insert_only.out
@@ -19,3 +19,12 @@
 4	D
 5	E
 
+-- !count_1 --
+4
+
+-- !count_2 --
+5
+
+-- !count_3 --
+5
+
diff --git a/regression-test/suites/external_table_p0/hive/test_transactional_hive.groovy b/regression-test/suites/external_table_p0/hive/test_transactional_hive.groovy
index 4f7008ec1726fa7..a12ab8a4f78cccf 100644
--- a/regression-test/suites/external_table_p0/hive/test_transactional_hive.groovy
+++ b/regression-test/suites/external_table_p0/hive/test_transactional_hive.groovy
@@ -114,6 +114,14 @@ suite("test_transactional_hive", "p0,external,hive,external_docker,external_dock
 
         }
     }
+    
+    def test_acid_count = {
+        qt_count_1 """ select count(*) from orc_full_acid; """ // 3 
+        qt_count_2 """ select count(*) from orc_full_acid_par; """  // 6
+        qt_count_3 """ select count(*) from orc_to_acid_compacted_tb; """ //4
+        qt_count_4 """ select count(*) from orc_acid_minor; """ //3
+        qt_count_5 """ select count(*) from orc_acid_major; """ //3
+    }
 
 
     String enabled = context.config.otherConfigs.get("enableHiveTest")
@@ -148,6 +156,10 @@ suite("test_transactional_hive", "p0,external,hive,external_docker,external_dock
             test_acid()
             test_acid_write()
 
+
+            test_acid_count()
+            
+            
             sql """drop catalog if exists ${catalog_name}"""
         } finally {
         }
diff --git a/regression-test/suites/external_table_p2/hive/test_hive_translation_insert_only.groovy b/regression-test/suites/external_table_p2/hive/test_hive_translation_insert_only.groovy
index 758417c32373b5f..f7135175152ce9a 100644
--- a/regression-test/suites/external_table_p2/hive/test_hive_translation_insert_only.groovy
+++ b/regression-test/suites/external_table_p2/hive/test_hive_translation_insert_only.groovy
@@ -45,6 +45,11 @@ suite("test_hive_translation_insert_only", "p2,external,hive,external_remote,ext
     qt_2 """ select * from parquet_insert_only_major order by id """ 
     qt_3 """ select * from orc_insert_only_minor order by id """ 
 
+    qt_count_1 """ select count(*) from text_insert_only """ //4 
+    qt_count_2 """ select count(*) from parquet_insert_only_major """ //5 
+    qt_count_3 """ select count(*) from orc_insert_only_minor """ //5
+
+
     sql """drop catalog ${hms_catalog_name};"""
 }
 

From 43f619725d409f4c4aad69c73e6b41eacb3a43e9 Mon Sep 17 00:00:00 2001
From: Xinyi Zou <zouxinyi@selectdb.com>
Date: Fri, 10 Jan 2025 19:16:02 +0800
Subject: [PATCH 109/140] [fix](memory) Fix reset Jemalloc `dirty_decay_ms`
 failed (#46710)

### What problem does this PR solve?

Introduced pr: #46136
fix:
```
start BE in local mode
*** Query id: 0-0 ***
*** is nereids: 0 ***
*** tablet id: 0 ***
*** Aborted at 1736203429 (unix time) try "date -d @1736203429" if you are using GNU date ***
*** Current BE git commitID: 5ee7e73313 ***
*** SIGSEGV address not mapped to object (@0x7b228e) received by PID 4568 (TID 6464 OR 0x7fd390c11640) from PID 8069774; stack trace: ***
 0# doris::signal::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*) at /doris/be/src/common/signal_handler.h:421
 1# PosixSignals::chained_handler(int, siginfo*, void*) [clone .part.0] in /usr/lib/jvm/java-17-openjdk-amd64/lib/server/libjvm.so
 2# JVM_handle_linux_signal in /usr/lib/jvm/java-17-openjdk-amd64/lib/server/libjvm.so
 3# 0x00007FD7DC196520 in /lib/x86_64-linux-gnu/libc.so.6
 4# je_pac_decay_ms_get at ../src/pac.c:555
 5# arena_i_dirty_decay_ms_ctl at ../src/ctl.c:2870
 6# je_ctl_byname at ../src/ctl.c:1574
 7# void doris::MemInfo::set_jemallctl_value<long>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, long) at /doris/be/src/util/mem_info.h:123
 8# doris::Daemon::je_reset_dirty_decay_thread() const at /doris/be/src/common/daemon.cpp:533
 9# doris::Thread::supervise_thread(void*) at /doris/be/src/util/thread.cpp:499
10# start_thread at ./nptl/pthread_create.c:442
11# 0x00007FD7DC27A850 at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:83
```
---
 be/src/util/mem_info.h | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/be/src/util/mem_info.h b/be/src/util/mem_info.h
index 5d1d0c75367673a..3fa0fdbd709209d 100644
--- a/be/src/util/mem_info.h
+++ b/be/src/util/mem_info.h
@@ -107,6 +107,8 @@ class MemInfo {
         size_t value_size = sizeof(T);
         if (jemallctl(name.c_str(), &value, &value_size, nullptr, 0) == 0) {
             return value;
+        } else {
+            LOG(WARNING) << fmt::format("Failed, jemallctl get {}", name);
         }
 #endif
         return 0;
@@ -123,9 +125,6 @@ class MemInfo {
             if (err) {
                 LOG(WARNING) << fmt::format("Failed, jemallctl value for {} set to {} (old {})",
                                             name, value, old_value);
-            } else {
-                LOG(INFO) << fmt::format("Successfully, jemallctl value for {} set to {} (old {})",
-                                         name, value, old_value);
             }
         } catch (...) {
             LOG(WARNING) << fmt::format("Exception, jemallctl value for {} set to {} (old {})",
@@ -140,8 +139,6 @@ class MemInfo {
             int err = jemallctl(name.c_str(), nullptr, nullptr, nullptr, 0);
             if (err) {
                 LOG(WARNING) << fmt::format("Failed, jemallctl action {}", name);
-            } else {
-                LOG(INFO) << fmt::format("Successfully, jemallctl action {}", name);
             }
         } catch (...) {
             LOG(WARNING) << fmt::format("Exception, jemallctl action {}", name);
@@ -181,8 +178,12 @@ class MemInfo {
         // Each time this interface is set, all currently unused dirty pages are considered
         // to have fully decayed, which causes immediate purging of all unused dirty pages unless
         // the decay time is set to -1
-        set_jemallctl_value<ssize_t>(fmt::format("arena.{}.dirty_decay_ms", MALLCTL_ARENAS_ALL),
-                                     dirty_decay_ms);
+        //
+        // NOTE: Using "arena.MALLCTL_ARENAS_ALL.dirty_decay_ms" to modify all arenas will fail or even crash,
+        // which may be a bug.
+        for (unsigned i = 0; i < get_jemallctl_value<unsigned>("arenas.narenas"); i++) {
+            set_jemallctl_value<ssize_t>(fmt::format("arena.{}.dirty_decay_ms", i), dirty_decay_ms);
+        }
 #endif
     }
 

From 8d1a643fc13b4397db0374cf671381724aa87972 Mon Sep 17 00:00:00 2001
From: Xin Liao <liaoxin@selectdb.com>
Date: Fri, 10 Jan 2025 22:37:47 +0800
Subject: [PATCH 110/140] [chore](load) Set the log level to debug and optimize
 the content for load job (#46683)

---
 .../main/java/org/apache/doris/load/loadv2/LoadJob.java    | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java
index 249756b501bc167..5faebd047c81a07 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java
@@ -1243,8 +1243,11 @@ public boolean isExpired(long currentTimeMs) {
             expireTime = Config.streaming_label_keep_max_second;
         }
 
-        LOG.info("state {}, expireTime {}, currentTimeMs {}, finishTimestamp {}",
-                state, expireTime, currentTimeMs, getFinishTimestamp());
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("Job ID: {}, DB ID: {}, Label: {}, State: {}, Expire Time: {}, Current Time: {}, "
+                      + "Finish Timestamp: {}", id, dbId, label, state, expireTime, currentTimeMs,
+                      getFinishTimestamp());
+        }
         return (currentTimeMs - getFinishTimestamp()) / 1000 > expireTime;
     }
 

From 8deca02bd3f1309a0ed302d8a12cb8208384a35d Mon Sep 17 00:00:00 2001
From: hui lai <laihui@selectdb.com>
Date: Fri, 10 Jan 2025 22:51:49 +0800
Subject: [PATCH 111/140] [fix](test) adapt the test_routine_load_with_user
 case to storage vaults (#46759)

---
 .../routine_load/test_routine_load_with_user.groovy | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_with_user.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_with_user.groovy
index 7b01dbacc6fa2d2..73cce57822fab3f 100644
--- a/regression-test/suites/load_p0/routine_load/test_routine_load_with_user.groovy
+++ b/regression-test/suites/load_p0/routine_load/test_routine_load_with_user.groovy
@@ -66,6 +66,19 @@ suite("test_routine_load_with_user","p0") {
         assertTrue(!clusters.isEmpty())
         def validCluster = clusters[0][0]
         sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}""";
+
+        try {
+            def storageVaults = (sql " SHOW STORAGE VAULT; ").stream().map(row -> row[0]).collect(Collectors.toSet())
+            logger.info("all vaults: ${storageVaults}")
+            for (String vault in storageVaults) {
+                sql """
+                    GRANT usage_priv ON storage vault ${vault} TO '${user}';
+                """
+            }
+        } catch (Exception e) {
+            // cloud instance may doesn't support storage vault
+            logger.info(e.getMessage())
+        }
     }
 
     connect(user, "${pwd}", context.config.jdbcUrl) {

From b1700b9ad0a60675f45ada0c1be64afb1d446e07 Mon Sep 17 00:00:00 2001
From: shuke <shuke@selectdb.com>
Date: Sat, 11 Jan 2025 10:52:13 +0800
Subject: [PATCH 112/140] [regression-test](fix) schema change case bug
 (#46782)

---
 .../schema_change_modify_mv_column_type_agg.out                 | 2 +-
 .../modify_col_type_dup/schema_change_modify_mv_column_type.out | 2 +-
 .../schema_change_modify_mv_column_type2.out                    | 2 +-
 .../schema_change_modify_mv_column_type_agg.groovy              | 2 +-
 .../schema_change_modify_mv_column_type.groovy                  | 2 +-
 .../schema_change_modify_mv_column_type2.groovy                 | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/regression-test/data/schema_change_p0/modify_col_type_agg/schema_change_modify_mv_column_type_agg.out b/regression-test/data/schema_change_p0/modify_col_type_agg/schema_change_modify_mv_column_type_agg.out
index c86341a56862c3e..b1feac9a13a341f 100644
--- a/regression-test/data/schema_change_p0/modify_col_type_agg/schema_change_modify_mv_column_type_agg.out
+++ b/regression-test/data/schema_change_p0/modify_col_type_agg/schema_change_modify_mv_column_type_agg.out
@@ -28,7 +28,7 @@ mv_tbl_scalar_types_agg_1	AGG_KEYS	mv_k2	bigint	bigint	Yes	true	\N		true	`k2`
 
 -- !sql --
 
--- !sql --
+-- !master_sql --
 tbl_scalar_types_agg	AGG_KEYS	k1	bigint	bigint	Yes	true	\N		true		
 		k2	bigint	bigint	Yes	true	\N		true		
 		c_bool	boolean	boolean	Yes	false	\N	REPLACE	true		
diff --git a/regression-test/data/schema_change_p0/modify_col_type_dup/schema_change_modify_mv_column_type.out b/regression-test/data/schema_change_p0/modify_col_type_dup/schema_change_modify_mv_column_type.out
index e7d4c6754770489..3503eca20850cff 100644
--- a/regression-test/data/schema_change_p0/modify_col_type_dup/schema_change_modify_mv_column_type.out
+++ b/regression-test/data/schema_change_p0/modify_col_type_dup/schema_change_modify_mv_column_type.out
@@ -62,7 +62,7 @@ mv_tbl_scalar_types_dup_1	DUP_KEYS	mv_c_tinyint	tinyint	tinyint	Yes	true	\N		tru
 -2106969609	true	10	29572	16738	1736115820	-957295886	-13319.206	-1.333603562816737E9	91224478600376111.942	69457425159617037.453	2022-09-06	2022-05-08T19:52:36	2022-04-05	2022-08-17T19:23:31	222.79.139.99	WalterFox@Voomm.net	Oxford Alley 77
 -2102307005	true	10	-23674	24613	-1810828490	-47095409	-14686.167	2.072108685694799E9	39847820962230526.125	584354832299375.156	2022-03-27	2022-02-11T13:46:06	2022-12-25	2022-11-28T09:37:49	213.146.33.250	JuliaSimmons@Zazio.info	Eagle Crest Terrace 84
 
--- !sql --
+-- !master_sql --
 tbl_scalar_types_dup	DUP_KEYS	k1	bigint	bigint	Yes	true	\N		true		
 		c_bool	boolean	boolean	Yes	false	\N	NONE	true		
 		c_tinyint	tinyint	tinyint	Yes	false	\N	NONE	true		
diff --git a/regression-test/data/schema_change_p0/modify_col_type_dup2/schema_change_modify_mv_column_type2.out b/regression-test/data/schema_change_p0/modify_col_type_dup2/schema_change_modify_mv_column_type2.out
index 5c0ee14eb32667b..024ed9d867e2019 100644
--- a/regression-test/data/schema_change_p0/modify_col_type_dup2/schema_change_modify_mv_column_type2.out
+++ b/regression-test/data/schema_change_p0/modify_col_type_dup2/schema_change_modify_mv_column_type2.out
@@ -1,5 +1,5 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
--- !sql --
+-- !master_sql --
 tbl_scalar_types_dup	DUP_KEYS	k1	bigint	bigint	Yes	true	\N		true		
 		c_bool	boolean	boolean	Yes	false	\N	NONE	true		
 		c_tinyint	tinyint	tinyint	Yes	false	\N	NONE	true		
diff --git a/regression-test/suites/schema_change_p0/modify_col_type_agg/schema_change_modify_mv_column_type_agg.groovy b/regression-test/suites/schema_change_p0/modify_col_type_agg/schema_change_modify_mv_column_type_agg.groovy
index c1613c696251a50..dd5cc5c47ee138f 100644
--- a/regression-test/suites/schema_change_p0/modify_col_type_agg/schema_change_modify_mv_column_type_agg.groovy
+++ b/regression-test/suites/schema_change_p0/modify_col_type_agg/schema_change_modify_mv_column_type_agg.groovy
@@ -92,6 +92,6 @@ suite("schema_change_modify_mv_column_type_agg") {
             }
         }
     }
-    qt_sql """ desc ${testTable} all """
+    qt_master_sql """ desc ${testTable} all """
     sql "INSERT INTO ${testTable} SELECT * from ${testTable}"
 }
diff --git a/regression-test/suites/schema_change_p0/modify_col_type_dup/schema_change_modify_mv_column_type.groovy b/regression-test/suites/schema_change_p0/modify_col_type_dup/schema_change_modify_mv_column_type.groovy
index 3ea57fefff573e1..92c3870c56339be 100644
--- a/regression-test/suites/schema_change_p0/modify_col_type_dup/schema_change_modify_mv_column_type.groovy
+++ b/regression-test/suites/schema_change_p0/modify_col_type_dup/schema_change_modify_mv_column_type.groovy
@@ -95,6 +95,6 @@ suite("schema_change_modify_mv_column_type") {
     }
     // sync materialized view rewrite will fail when schema change, tmp disable, enable when fixed
     sql """set enable_dml_materialized_view_rewrite = false;"""
-    qt_sql """ desc ${testTable} all """
+    qt_master_sql """ desc ${testTable} all """
     sql "INSERT INTO ${testTable} SELECT * from ${testTable}"
 }
diff --git a/regression-test/suites/schema_change_p0/modify_col_type_dup2/schema_change_modify_mv_column_type2.groovy b/regression-test/suites/schema_change_p0/modify_col_type_dup2/schema_change_modify_mv_column_type2.groovy
index c2f7caa1af4f5fd..2f475992b494372 100644
--- a/regression-test/suites/schema_change_p0/modify_col_type_dup2/schema_change_modify_mv_column_type2.groovy
+++ b/regression-test/suites/schema_change_p0/modify_col_type_dup2/schema_change_modify_mv_column_type2.groovy
@@ -69,7 +69,7 @@ suite("schema_change_modify_mv_column_type2") {
         }
     }
     createMV ("""CREATE MATERIALIZED VIEW mv_${testTable}_2 AS SELECT k1, sum(c_int), max(c_int), min(c_int) FROM ${testTable} GROUP BY k1""")
-    qt_sql """ desc ${testTable} all """
+    qt_master_sql """ desc ${testTable} all """
     sql "set topn_opt_limit_threshold = 100"
     qt_sql "SELECT * from ${testTable} order by 1, 2, 3 limit 10"
     qt_sql "SELECT * from ${testTable} where c_tinyint = 10 order by 1, 2, 3 limit 10 "

From 46bd885200aefa5bbd8a78b644b50ae5989edcbf Mon Sep 17 00:00:00 2001
From: shuke <shuke@selectdb.com>
Date: Sat, 11 Jan 2025 10:52:23 +0800
Subject: [PATCH 113/140] =?UTF-8?q?[regression-test](fix)=20fix=20test=5Fs?=
 =?UTF-8?q?how=5Ftablet.groovy=20bug=20running=20on=20mul=E2=80=A6=20(#467?=
 =?UTF-8?q?91)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../suites/show_p0/test_show_tablet.groovy    | 32 +++++++++++++++----
 1 file changed, 25 insertions(+), 7 deletions(-)

diff --git a/regression-test/suites/show_p0/test_show_tablet.groovy b/regression-test/suites/show_p0/test_show_tablet.groovy
index 59e7c1a8e4da2f2..024e90fdd198bf0 100644
--- a/regression-test/suites/show_p0/test_show_tablet.groovy
+++ b/regression-test/suites/show_p0/test_show_tablet.groovy
@@ -25,14 +25,32 @@ suite("test_show_tablet") {
             PROPERTIES (
                 "replication_num" = "1"
             );"""
-    def res = sql """SHOW TABLETS FROM show_tablets_test_t limit 5, 1;"""
 
-    logger.info("result: " + res.toString());
-    assertTrue(res.size() == 0)
+    def res = sql """ SHOW TABLETS FROM show_tablets_test_t """
+    if (res.size() == 5) {
+        // replication num == 1
+        res = sql """SHOW TABLETS FROM show_tablets_test_t limit 5, 1;"""
+        logger.info("result: " + res.toString());
+        assertTrue(res.size() == 0)
 
-    res = sql """SHOW TABLETS FROM show_tablets_test_t limit 3, 5;"""
-    assertTrue(res.size() == 2)
+        res = sql """SHOW TABLETS FROM show_tablets_test_t limit 3, 5;"""
+        assertTrue(res.size() == 2)
 
-    res = sql """SHOW TABLETS FROM show_tablets_test_t limit 10;"""
-    assertTrue(res.size() == 5)
+        res = sql """SHOW TABLETS FROM show_tablets_test_t limit 10;"""
+        assertTrue(res.size() == 5)
+    } else if (res.size() == 15) {
+        // in multi-be cluster and force_olap_table_replication_num=3
+        // will change to 3 replication even though set "replication_num" = "1" in create table
+        res = sql """SHOW TABLETS FROM show_tablets_test_t limit 15, 1;"""
+        logger.info("result: " + res.toString());
+        assertTrue(res.size() == 0)
+
+        res = sql """SHOW TABLETS FROM show_tablets_test_t limit 13, 5;"""
+        assertTrue(res.size() == 2)
+
+        res = sql """SHOW TABLETS FROM show_tablets_test_t limit 15;"""
+        assertTrue(res.size() == 15)
+    } else {
+        assertTrue(1 == 2)
+    }
 }
\ No newline at end of file

From 294fe86968bb90338186548435b436f11f21204c Mon Sep 17 00:00:00 2001
From: Dongyang Li <lidongyang@selectdb.com>
Date: Sat, 11 Jan 2025 10:53:37 +0800
Subject: [PATCH 114/140] [fix](case) fix defined global variables in script
 (#46806)

---
 regression-test/suites/ssb_sf0.1_p1/load.groovy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/regression-test/suites/ssb_sf0.1_p1/load.groovy b/regression-test/suites/ssb_sf0.1_p1/load.groovy
index 3b3955dc3ac7e63..240bd8a1396ed09 100644
--- a/regression-test/suites/ssb_sf0.1_p1/load.groovy
+++ b/regression-test/suites/ssb_sf0.1_p1/load.groovy
@@ -89,7 +89,7 @@ suite("load") {
         sql new File("""${context.file.parent}/ddl/${table}_delete.sql""").text
         sql "set insert_timeout=3600"
         def r = sql "select @@insert_timeout"
-        year_cons = [
+        def year_cons = [
             'lo_orderdate<19930101',
             'lo_orderdate>=19930101 and lo_orderdate<19940101',
             'lo_orderdate>=19940101 and lo_orderdate<19950101',

From 84d5b98edac32cee579fd80a40a2e3d349255949 Mon Sep 17 00:00:00 2001
From: shuke <shuke@selectdb.com>
Date: Sat, 11 Jan 2025 11:18:49 +0800
Subject: [PATCH 115/140] [regression-test](fix) fix 'desc table' to follower
 assertion failure (#46788)

---
 .../create_nestedtypes_with_schemachange.out  | 24 +++++++++----------
 ...reate_nestedtypes_with_schemachange.groovy |  4 ++--
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/regression-test/data/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.out b/regression-test/data/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.out
index 5b811095fcbc336..efcecd7595305a1 100644
--- a/regression-test/data/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.out
+++ b/regression-test/data/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.out
@@ -1,5 +1,5 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
--- !sql --
+-- !master_sql --
 col0	bigint	No	true	\N	
 col_array	array<text>	Yes	false	\N	NONE
 col2	int	No	false	\N	NONE
@@ -7,7 +7,7 @@ col3	array<int>	Yes	false	\N	NONE
 col4	map<int,int>	Yes	false	\N	NONE
 col5	struct<f1:int>	Yes	false	\N	NONE
 
--- !sql --
+-- !master_sql --
 col0	bigint	No	true	\N	
 col_map	map<char(32),text>	Yes	false	\N	NONE
 col2	int	No	false	\N	NONE
@@ -15,7 +15,7 @@ col3	array<int>	Yes	false	\N	NONE
 col4	map<int,int>	Yes	false	\N	NONE
 col5	struct<f1:int>	Yes	false	\N	NONE
 
--- !sql --
+-- !master_sql --
 col0	bigint	No	true	\N	
 col_struct	struct<f1:varchar(1)>	Yes	false	\N	NONE
 col2	int	No	false	\N	NONE
@@ -23,21 +23,21 @@ col3	array<int>	Yes	false	\N	NONE
 col4	map<int,int>	Yes	false	\N	NONE
 col5	struct<f1:int>	Yes	false	\N	NONE
 
--- !sql --
+-- !master_sql --
 col0	bigint	No	true	\N	
 col2	int	No	false	\N	NONE
 col3	array<int>	Yes	false	\N	NONE
 col4	map<int,int>	Yes	false	\N	NONE
 col5	struct<f1:int>	Yes	false	\N	NONE
 
--- !sql --
+-- !master_sql --
 col0	bigint	No	true	\N	
 col2	int	No	false	\N	NONE
 col3	array<int>	Yes	false	\N	NONE
 col4	map<int,int>	Yes	false	\N	NONE
 col5	struct<f1:int>	Yes	false	\N	NONE
 
--- !sql --
+-- !master_sql --
 col0	bigint	No	true	\N	
 col2	int	No	false	\N	NONE
 col3	array<int>	Yes	false	\N	NONE
@@ -47,7 +47,7 @@ col5	struct<f1:int>	Yes	false	\N	NONE
 -- !sql_before --
 1	2	[1, 2]	{1:2}	{"f1":1}	{"a":[1,2,3]}
 
--- !sql --
+-- !master_sql --
 col0	bigint	No	true	\N	
 col2	int	No	false	\N	NONE
 col3	array<int>	Yes	false	\N	NONE
@@ -61,7 +61,7 @@ col6	variant	Yes	false	\N	NONE
 -- !sql_before --
 1	2	[1, 2]	{1:2}	{"f1":1}	{"a":[1,2,3]}
 
--- !sql --
+-- !master_sql --
 col0	bigint	No	true	\N	
 col2	int	No	false	\N	NONE
 col3	array<int>	Yes	false	\N	NONE
@@ -75,7 +75,7 @@ col6	variant	No	false	\N	NONE
 -- !sql_before --
 1	2	[1, 2]	{1:2}	{"f1":1}	{"a":[1,2,3]}
 
--- !sql --
+-- !master_sql --
 col0	bigint	No	true	\N	
 col2	int	No	false	\N	NONE
 col3	array<int>	Yes	false	\N	NONE
@@ -89,7 +89,7 @@ col6	variant	Yes	false	\N	NONE
 -- !sql_before --
 1	2	[1, 2]	{1:2}	{"f1":1}	{"a":[1,2,3]}
 
--- !sql --
+-- !master_sql --
 col0	bigint	No	true	\N	
 col2	int	No	false	\N	NONE
 col3	array<int>	No	false	\N	NONE
@@ -103,7 +103,7 @@ col6	variant	No	false	\N	NONE
 -- !sql_before --
 1	2	[1, 2]	{1:2}	{"f1":1}	{"a":[1,2,3]}
 
--- !sql --
+-- !master_sql --
 col0	bigint	No	true	\N	
 col2	int	No	false	\N	NONE
 col3	array<int>	Yes	false	\N	NONE
@@ -117,7 +117,7 @@ col6	variant	Yes	false	\N	NONE
 -- !sql_before --
 1	2	[1, 2]	{1:2}	{"f1":1}	{"a":[1,2,3]}
 
--- !sql --
+-- !master_sql --
 col0	bigint	No	true	\N	
 col2	int	No	false	\N	NONE
 col3	array<int>	No	false	\N	NONE
diff --git a/regression-test/suites/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.groovy b/regression-test/suites/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.groovy
index 62b9ab3bb567f47..26db35cbbe04bca 100644
--- a/regression-test/suites/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.groovy
+++ b/regression-test/suites/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.groovy
@@ -45,7 +45,7 @@ suite("create_nestedtypes_with_schemachange", "p0") {
             }
         }
         // desc table
-        qt_sql "DESC $testTablex"
+        qt_master_sql "DESC $testTablex"
     }
 
     // array
@@ -102,7 +102,7 @@ suite("create_nestedtypes_with_schemachange", "p0") {
             }
         }
         // desc table
-        qt_sql "DESC $testTablex"
+        qt_master_sql "DESC $testTablex"
         qt_sql_after "select * from $testTablex"
     }
 

From ec6d846a0ee73d93b142bcd2dcb75663814639c0 Mon Sep 17 00:00:00 2001
From: Dongyang Li <lidongyang@selectdb.com>
Date: Sat, 11 Jan 2025 11:50:26 +0800
Subject: [PATCH 116/140] [fix](case) fix some failed cases (#46807)

---
 .../suites/inject_hdfs_vault_p0/inject_hdfs_load_error.groovy | 4 ++++
 .../inject_hdfs_vault_p0/inject_hdfs_select_error.groovy      | 4 ++++
 .../test_index_compaction_with_multi_index_segments.groovy    | 2 +-
 .../inverted_index_p0/test_inverted_index_file_size.groovy    | 2 +-
 .../inverted_index_p0/test_inverted_index_v2_file_size.groovy | 4 ++--
 regression-test/suites/node_p0/test_frontend.groovy           | 2 +-
 6 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/regression-test/suites/inject_hdfs_vault_p0/inject_hdfs_load_error.groovy b/regression-test/suites/inject_hdfs_vault_p0/inject_hdfs_load_error.groovy
index 407f914f96466e8..cc24c06e4093c8a 100755
--- a/regression-test/suites/inject_hdfs_vault_p0/inject_hdfs_load_error.groovy
+++ b/regression-test/suites/inject_hdfs_vault_p0/inject_hdfs_load_error.groovy
@@ -18,6 +18,10 @@
 import org.codehaus.groovy.runtime.IOGroovyMethods
 
 suite("inject_hdfs_load_error", "nonConcurrent") {
+    if (!isCloudMode()) {
+        logger.info("skip create storgage vault case")
+        return
+    }
     if (!enableStoragevault()) {
         logger.info("skip create storgage vault case")
         return
diff --git a/regression-test/suites/inject_hdfs_vault_p0/inject_hdfs_select_error.groovy b/regression-test/suites/inject_hdfs_vault_p0/inject_hdfs_select_error.groovy
index e49a9a242b2d9cb..c993df538d0b6bc 100755
--- a/regression-test/suites/inject_hdfs_vault_p0/inject_hdfs_select_error.groovy
+++ b/regression-test/suites/inject_hdfs_vault_p0/inject_hdfs_select_error.groovy
@@ -18,6 +18,10 @@
 import org.codehaus.groovy.runtime.IOGroovyMethods
 
 suite("inject_hdfs_load_error", "nonConcurrent") {
+    if (!isCloudMode()) {
+        logger.info("skip create storgage vault case")
+        return
+    }
     if (!enableStoragevault()) {
         logger.info("skip create storgage vault case")
         return
diff --git a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy
index 5d9c53ccfb23d79..6b8b64ec11e1537 100644
--- a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy
+++ b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy
@@ -288,7 +288,7 @@ suite("test_index_compaction_with_multi_index_segments", "nonConcurrent") {
         assert (rowsetCount == 3 * replicaNum)
 
         // trigger full compactions for all tablets in ${tableName}
-        trigger_and_wait_compaction.call(tableName, "full")
+        trigger_and_wait_compaction(tableName, "full")
         // after full compaction, there is only 1 rowset.
         rowsetCount = get_rowset_count.call(tablets)
         if (isCloudMode) {
diff --git a/regression-test/suites/inverted_index_p0/test_inverted_index_file_size.groovy b/regression-test/suites/inverted_index_p0/test_inverted_index_file_size.groovy
index 73446089bb209ce..b0fee7cc5579bc4 100644
--- a/regression-test/suites/inverted_index_p0/test_inverted_index_file_size.groovy
+++ b/regression-test/suites/inverted_index_p0/test_inverted_index_file_size.groovy
@@ -87,7 +87,7 @@ suite("test_inverted_index_file_size", "nonConcurrent"){
 
         qt_sql """ select count() from ${tableName} where clientip match '17.0.0.0' and request match 'GET' and status match '200' and size > 200 """
         qt_sql """ select count() from ${tableName} where clientip match_phrase '17.0.0.0' and request match_phrase 'GET' and status match '200' and size > 200 """
-        trigger_and_wait_compaction.call(tableName, "full")
+        trigger_and_wait_compaction(tableName, "full")
         qt_sql """ select count() from ${tableName} where clientip match '17.0.0.0' and request match 'GET' and status match '200' and size > 200 """
         qt_sql """ select count() from ${tableName} where clientip match_phrase '17.0.0.0' and request match_phrase 'GET' and status match '200' and size > 200 """
 
diff --git a/regression-test/suites/inverted_index_p0/test_inverted_index_v2_file_size.groovy b/regression-test/suites/inverted_index_p0/test_inverted_index_v2_file_size.groovy
index e9ab66bbad40b13..76017c8bb305656 100644
--- a/regression-test/suites/inverted_index_p0/test_inverted_index_v2_file_size.groovy
+++ b/regression-test/suites/inverted_index_p0/test_inverted_index_v2_file_size.groovy
@@ -100,7 +100,7 @@ suite("test_index_index_V2_file_size", "nonConcurrent") {
         qt_sql """ select * from ${tableName} where score < 100 order by id, name, hobbies, score """
 
         // trigger full compactions for all tablets in ${tableName}
-        trigger_and_wait_compaction.call(tableName, "full")
+        trigger_and_wait_compaction(tableName, "full")
 
         def dedup_tablets = deduplicate_tablets(tablets)
 
@@ -135,7 +135,7 @@ suite("test_index_index_V2_file_size", "nonConcurrent") {
 
         set_be_config.call("inverted_index_compaction_enable", "false")
         // trigger full compactions for all tablets in ${tableName}
-        trigger_and_wait_compaction.call(tableName, "full")
+        trigger_and_wait_compaction(tableName, "full")
 
         // after full compaction, there is only 1 rowset.
         count = get_rowset_count.call(tablets);
diff --git a/regression-test/suites/node_p0/test_frontend.groovy b/regression-test/suites/node_p0/test_frontend.groovy
index 29ee3e06315a303..57d1445e63db319 100644
--- a/regression-test/suites/node_p0/test_frontend.groovy
+++ b/regression-test/suites/node_p0/test_frontend.groovy
@@ -22,7 +22,7 @@ suite("test_frontend", "nonConcurrent") {
     def res2 = sql """SHOW FRONTENDS Disks"""
     assertTrue(res2.size() != 0)
 
-    if (Config.isCloudMode()) {
+    if (isCloudMode()) {
         // In the test_sql_mode_node_mgr regression case, there is already a similar and more complex case. This case is redundant. Additionally, there is a 5-minute limit for dropping FE on the cloud.
         // so ignore it in cloud
         return;

From 4cf70c0a8ebf8217ec8aad4577e8114c8a92efc3 Mon Sep 17 00:00:00 2001
From: hui lai <laihui@selectdb.com>
Date: Sat, 11 Jan 2025 12:24:22 +0800
Subject: [PATCH 117/140] [fix](test) fix memtable flush regression test fail
 (#46749)

---
 ..._memtable_flush_is_high_priority_for_vtablet_writerV2.groovy | 2 +-
 ...is_high_priority_for_vtablet_writerv1_fault_injection.groovy | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/regression-test/suites/fault_injection_p0/test_memtable_flush_is_high_priority_for_vtablet_writerV2.groovy b/regression-test/suites/fault_injection_p0/test_memtable_flush_is_high_priority_for_vtablet_writerV2.groovy
index e91a06e2cf0b4c6..42e9fb865e85365 100644
--- a/regression-test/suites/fault_injection_p0/test_memtable_flush_is_high_priority_for_vtablet_writerV2.groovy
+++ b/regression-test/suites/fault_injection_p0/test_memtable_flush_is_high_priority_for_vtablet_writerV2.groovy
@@ -58,7 +58,7 @@ suite("test_memtable_flush_is_high_priority_for_vtablet_writerV2", "nonConcurren
         qt_sql """select * from ${testTable} order by id"""
         sql """set enable_memtable_on_sink_node = ${original_status}"""
     } catch(Exception e) {
-        log.error(e.message())
+        log.error(e.getMessage())
     } finally {
         GetDebugPoint().disableDebugPointForAllBEs("VTabletWriterV2._init.is_high_priority")
     }
diff --git a/regression-test/suites/fault_injection_p0/test_memtable_flush_is_high_priority_for_vtablet_writerv1_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_memtable_flush_is_high_priority_for_vtablet_writerv1_fault_injection.groovy
index b9fd8ecf5517001..d1e18ed0a63e6d6 100644
--- a/regression-test/suites/fault_injection_p0/test_memtable_flush_is_high_priority_for_vtablet_writerv1_fault_injection.groovy
+++ b/regression-test/suites/fault_injection_p0/test_memtable_flush_is_high_priority_for_vtablet_writerv1_fault_injection.groovy
@@ -58,7 +58,7 @@ suite("test_memtable_flush_is_high_priority_for_vtablet_writerV1", "nonConcurren
         qt_sql """select * from ${testTable} order by id"""
         sql """set enable_memtable_on_sink_node = ${original_status}"""
     } catch(Exception e) {
-        log.error(e.message())
+        log.error(e.getMessage())
     } finally {
         GetDebugPoint().disableDebugPointForAllBEs("VTabletWriter._init.is_high_priority")
     }

From 9fc6b8d2f9a4cf5d60b19ec177925680a358e97c Mon Sep 17 00:00:00 2001
From: camby <cambyzhu@tencent.com>
Date: Sat, 11 Jan 2025 13:46:01 +0800
Subject: [PATCH 118/140] [feat](api) add nullable info for api _schema
 (#46557)

### What problem does this PR solve?
spark doris connector still use `/api/<db>/<table>/_schema` api to get
column info, here we need nullable information.

like information_schema.columns, we add property: `is_nullable = Yes or
No`
---
 .../java/org/apache/doris/httpv2/rest/TableSchemaAction.java     | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableSchemaAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableSchemaAction.java
index 0f362b4aad89e6b..ec97c2c241f25f6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableSchemaAction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableSchemaAction.java
@@ -109,6 +109,7 @@ protected Object schema(
                         Optional aggregationType = Optional.ofNullable(column.getAggregationType());
                         baseInfo.put("aggregation_type", aggregationType.isPresent()
                                 ? column.getAggregationType().toSql() : "");
+                        baseInfo.put("is_nullable", column.isAllowNull() ? "Yes" : "No");
                         propList.add(baseInfo);
                     }
                     resultMap.put("status", 200);

From 8fa7bade79efa289ed77e5e457521c1e4bbd2436 Mon Sep 17 00:00:00 2001
From: shuke <shuke@selectdb.com>
Date: Sat, 11 Jan 2025 21:20:36 +0800
Subject: [PATCH 119/140] [regression-test](fix) remove unused features in
 manager test (#46786)

---
 .../manager/test_manager_interface_4.groovy   | 37 +++----------------
 1 file changed, 6 insertions(+), 31 deletions(-)

diff --git a/regression-test/suites/manager/test_manager_interface_4.groovy b/regression-test/suites/manager/test_manager_interface_4.groovy
index 78d8d451b7b7964..5de11e2007b1f69 100644
--- a/regression-test/suites/manager/test_manager_interface_4.groovy
+++ b/regression-test/suites/manager/test_manager_interface_4.groovy
@@ -49,42 +49,17 @@ suite('test_manager_interface_4',"p0") {
 
         futures.add( thread {
             sleep(500);
-            List<List<Object>> result = sql """ show proc '/current_query_stmts' """ 
-            logger.info("result = ${result}" )
-            def x = 0
+            List<List<Object>> result = sql_return_maparray """  show processlist  """
             def queryid = ""
+            def x = 0
             logger.info("result = ${result}")
-
-            for( int i = 0;i<result.size();i++) {
-                if (result[i][7] != null && result[i][7].contains("18880094567") )//Statement
-                {
-                    x = 1
-                    queryid = result[i][0]
-                    logger.info("query ID = ${queryid}")
-                    assertTrue(result[i][0]!= null) //QueryId
-                    assertTrue(result[i][1]!= null) //ConnectionId
-                    assertTrue(result[i][2]!= null)//Catalog
-                    assertTrue(result[i][3]!= null)//Database
-                    assertTrue(result[i][4]!= null)//User
-                    assertTrue(result[i][5]!= null)//ExecTime
-                    assertTrue(result[i][5].isNumber())//ExecTime
-                    assertTrue(result[i][6]!= null)//SqlHash
-                }
-            }
-            assertTrue(x == 1)
-            
-            x = 0 
-            result = sql """  show proc '/current_queries' """
-            logger.info("result = ${result}")
-            for( int i = 0;i<result.size();i++) {
-                if (result[i][0] == queryid )//QueryId
-                {
+            for( int i =0 ;i < result.size();i++ ){
+                if (result[i]["Info"].contains("18880094567")) {
+                    queryid = result[i]["QueryId"]
                     x = 1
-                    assertTrue(result[i][5]!= null)//ScanBytes
-                    assertTrue(result[i][6]!= null)//ProcessBytes
+                    break;
                 }
             }
-            assertTrue(x == 1)
 
             result = sql """  show processlist  """
             logger.info("result = ${result}")

From 958ad031e9f4a5be094ad860458400b02957e6ed Mon Sep 17 00:00:00 2001
From: Chester <chesterxu@tencent.com>
Date: Sat, 11 Jan 2025 22:14:51 +0800
Subject: [PATCH 120/140] [refactor](opt) remove redundant
 use_default_implementation_for_nulls() of some BE codes (#46796)

---
 be/src/vec/functions/array/function_array_difference.h | 2 --
 be/src/vec/functions/function_hll.cpp                  | 2 --
 be/src/vec/functions/function_multi_same_args.h        | 2 --
 be/src/vec/functions/function_struct_element.cpp       | 2 --
 be/src/vec/functions/function_time_value_to_field.cpp  | 2 --
 be/src/vec/functions/function_totype.h                 | 2 --
 6 files changed, 12 deletions(-)

diff --git a/be/src/vec/functions/array/function_array_difference.h b/be/src/vec/functions/array/function_array_difference.h
index 283ac206ce69b75..149e311e9405be7 100644
--- a/be/src/vec/functions/array/function_array_difference.h
+++ b/be/src/vec/functions/array/function_array_difference.h
@@ -66,8 +66,6 @@ class FunctionArrayDifference : public IFunction {
 
     size_t get_number_of_arguments() const override { return 1; }
 
-    bool use_default_implementation_for_nulls() const override { return true; }
-
     DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
         DCHECK(is_array(arguments[0]))
                 << "argument for function: " << name << " should be DataTypeArray but it has type "
diff --git a/be/src/vec/functions/function_hll.cpp b/be/src/vec/functions/function_hll.cpp
index 1040d7ea33d71b1..950197361cbbee4 100644
--- a/be/src/vec/functions/function_hll.cpp
+++ b/be/src/vec/functions/function_hll.cpp
@@ -152,8 +152,6 @@ class FunctionHllFromBase64 : public IFunction {
 
     size_t get_number_of_arguments() const override { return 1; }
 
-    bool use_default_implementation_for_nulls() const override { return true; }
-
     Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
                         uint32_t result, size_t input_rows_count) const override {
         auto res_null_map = ColumnUInt8::create(input_rows_count, 0);
diff --git a/be/src/vec/functions/function_multi_same_args.h b/be/src/vec/functions/function_multi_same_args.h
index f22bae640aeda4a..5ceedfa11e9d756 100644
--- a/be/src/vec/functions/function_multi_same_args.h
+++ b/be/src/vec/functions/function_multi_same_args.h
@@ -34,8 +34,6 @@ class FunctionMultiSameArgs : public IFunction {
 
     String get_name() const override { return name; }
 
-    bool use_default_implementation_for_nulls() const override { return true; }
-
     bool is_variadic() const override { return true; }
 
     size_t get_number_of_arguments() const override { return 0; }
diff --git a/be/src/vec/functions/function_struct_element.cpp b/be/src/vec/functions/function_struct_element.cpp
index f547588dece6467..ef2dfeff49ae4ae 100644
--- a/be/src/vec/functions/function_struct_element.cpp
+++ b/be/src/vec/functions/function_struct_element.cpp
@@ -47,8 +47,6 @@ class FunctionStructElement : public IFunction {
     // Get function name.
     String get_name() const override { return name; }
 
-    bool use_default_implementation_for_nulls() const override { return true; }
-
     size_t get_number_of_arguments() const override { return 2; }
 
     ColumnNumbers get_arguments_that_are_always_constant() const override { return {1}; }
diff --git a/be/src/vec/functions/function_time_value_to_field.cpp b/be/src/vec/functions/function_time_value_to_field.cpp
index 8c8ec3cf9ab2199..ee963d6154bb701 100644
--- a/be/src/vec/functions/function_time_value_to_field.cpp
+++ b/be/src/vec/functions/function_time_value_to_field.cpp
@@ -47,8 +47,6 @@ class FunctionTimeValueToField : public IFunction {
         return std::make_shared<ToDataType>();
     }
 
-    bool use_default_implementation_for_nulls() const override { return true; }
-
     Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
                         uint32_t result, size_t input_rows_count) const override {
         DCHECK_EQ(arguments.size(), 1);
diff --git a/be/src/vec/functions/function_totype.h b/be/src/vec/functions/function_totype.h
index e5a2e6cfd7b8d34..4e3e6dec8a4f31d 100644
--- a/be/src/vec/functions/function_totype.h
+++ b/be/src/vec/functions/function_totype.h
@@ -462,8 +462,6 @@ class FunctionStringOperateToNullType : public IFunction {
         return make_nullable(std::make_shared<typename Impl::ReturnType>());
     }
 
-    bool use_default_implementation_for_nulls() const override { return true; }
-
     Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
                         uint32_t result, size_t input_rows_count) const override {
         auto null_map = ColumnUInt8::create(input_rows_count, 0);

From 5a349291a272b1cc06c4929c89eecd0679160a8b Mon Sep 17 00:00:00 2001
From: walter <maochuan@selectdb.com>
Date: Sun, 12 Jan 2025 16:14:42 +0800
Subject: [PATCH 121/140] [chore](io) Add debug log for critical file
 operations (#46770)

---
 be/src/io/fs/local_file_system.cpp | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/be/src/io/fs/local_file_system.cpp b/be/src/io/fs/local_file_system.cpp
index 9270d919a37519d..9b6e9d1f835c8f1 100644
--- a/be/src/io/fs/local_file_system.cpp
+++ b/be/src/io/fs/local_file_system.cpp
@@ -57,6 +57,8 @@ LocalFileSystem::~LocalFileSystem() = default;
 
 Status LocalFileSystem::create_file_impl(const Path& file, FileWriterPtr* writer,
                                          const FileWriterOptions* opts) {
+    VLOG_DEBUG << "create file: " << file.native()
+               << ", sync_data: " << (opts ? opts->sync_file_data : true);
     TEST_SYNC_POINT_RETURN_WITH_VALUE("LocalFileSystem::create_file_impl",
                                       Status::IOError("inject io error"));
     int fd = ::open(file.c_str(), O_TRUNC | O_WRONLY | O_CREAT | O_CLOEXEC, 0666);
@@ -108,6 +110,8 @@ Status LocalFileSystem::open_file_impl(const Path& file, FileReaderSPtr* reader,
 }
 
 Status LocalFileSystem::create_directory_impl(const Path& dir, bool failed_if_exists) {
+    VLOG_DEBUG << "create directory: " << dir.native()
+               << ", failed_if_exists: " << failed_if_exists;
     bool exists = true;
     RETURN_IF_ERROR(exists_impl(dir, &exists));
     if (exists && failed_if_exists) {
@@ -124,6 +128,7 @@ Status LocalFileSystem::create_directory_impl(const Path& dir, bool failed_if_ex
 }
 
 Status LocalFileSystem::delete_file_impl(const Path& file) {
+    VLOG_DEBUG << "delete file: " << file.native();
     bool exists = true;
     RETURN_IF_ERROR(exists_impl(file, &exists));
     if (!exists) {
@@ -141,6 +146,7 @@ Status LocalFileSystem::delete_file_impl(const Path& file) {
 }
 
 Status LocalFileSystem::delete_directory_impl(const Path& dir) {
+    VLOG_DEBUG << "delete directory: " << dir.native();
     bool exists = true;
     RETURN_IF_ERROR(exists_impl(dir, &exists));
     if (!exists) {
@@ -249,6 +255,7 @@ Status LocalFileSystem::list_impl(const Path& dir, bool only_file, std::vector<F
 }
 
 Status LocalFileSystem::rename_impl(const Path& orig_name, const Path& new_name) {
+    VLOG_DEBUG << "rename file: " << orig_name.native() << " to " << new_name.native();
     TEST_SYNC_POINT_RETURN_WITH_VALUE("LocalFileSystem::rename",
                                       Status::IOError("inject io error"));
     std::error_code ec;
@@ -265,6 +272,7 @@ Status LocalFileSystem::link_file(const Path& src, const Path& dest) {
 }
 
 Status LocalFileSystem::link_file_impl(const Path& src, const Path& dest) {
+    VLOG_DEBUG << "link file: " << src.native() << " to " << dest.native();
     if (::link(src.c_str(), dest.c_str()) != 0) {
         return localfs_error(errno, fmt::format("failed to create hard link from {} to {}",
                                                 src.native(), dest.native()));
@@ -364,6 +372,7 @@ Status LocalFileSystem::copy_path(const Path& src, const Path& dest) {
 }
 
 Status LocalFileSystem::copy_path_impl(const Path& src, const Path& dest) {
+    VLOG_DEBUG << "copy from " << src.native() << " to " << dest.native();
     std::error_code ec;
     std::filesystem::copy(src, dest, std::filesystem::copy_options::recursive, ec);
     if (ec) {

From 4f68b81ba72c44a1050a7176a9044c6da892c23c Mon Sep 17 00:00:00 2001
From: Uniqueyou <wangyixuan@selectdb.com>
Date: Sun, 12 Jan 2025 16:18:09 +0800
Subject: [PATCH 122/140] [Test](check) add check for materialized view
 (#46713)

---
 .../check_before_quit/check_before_quit.groovy    | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/regression-test/suites/check_before_quit/check_before_quit.groovy b/regression-test/suites/check_before_quit/check_before_quit.groovy
index 995c81f057917d6..ad35ce3db7106b0 100644
--- a/regression-test/suites/check_before_quit/check_before_quit.groovy
+++ b/regression-test/suites/check_before_quit/check_before_quit.groovy
@@ -253,10 +253,21 @@ suite("check_before_quit", "nonConcurrent,p0") {
         logger.info("show all tabkes: ${allTables}")
         for (int j = 0;j < allTables.size();j ++) {
             def tbl = allTables[j][0]
-            def createTableSql = sql "show create table ${db}.${tbl}"
+            def createTableSql = ""
+            try {
+                createTableSql = sql "show create table ${db}.${tbl}"
+                logger.info("create table sql: ${createTableSql}")
+            } catch (Exception e) {
+                if (e.getMessage().contains("not support async materialized view")) {
+                    createTableSql = sql "show create materialized view ${tbl}"
+                    logger.info("create materialized view sql: ${createTableSql}")
+                }
+            }
             if (createTableSql[0][1].contains("CREATE VIEW")) {
                 sql "drop view if exists ${tbl}"
-            }else {
+            } else if (createTableSql[0][1].contains("CREATE MATERIALIZED VIEW")) {
+                sql "drop materialized view if exists ${tbl}"
+            } else {
                 sql "drop table if exists ${tbl}"
             }
             sql(createTableSql[0][1])

From 95b5b0029f2df1f71a283160d98641a1f850ef2e Mon Sep 17 00:00:00 2001
From: Kaijie Chen <chenkaijie@selectdb.com>
Date: Sun, 12 Jan 2025 22:35:09 +0800
Subject: [PATCH 123/140] [debug](move-memtable) add debug logs for
 skip_two_backends injection cases (#46826)

---
 be/src/vec/sink/writer/vtablet_writer_v2.cpp | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/be/src/vec/sink/writer/vtablet_writer_v2.cpp b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
index f3ad6b1d5e3f0c6..c332e3bf2f2e7ff 100644
--- a/be/src/vec/sink/writer/vtablet_writer_v2.cpp
+++ b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
@@ -392,11 +392,21 @@ Status VTabletWriterV2::_select_streams(int64_t tablet_id, int64_t partition_id,
         VLOG_DEBUG << fmt::format("_select_streams P{} I{} T{}", partition_id, index_id, tablet_id);
         _tablets_for_node[node_id].emplace(tablet_id, tablet);
         auto stream = _load_stream_map->at(node_id)->select_one_stream();
+        DBUG_EXECUTE_IF("VTabletWriterV2._open_streams.skip_two_backends", {
+            LOG(INFO) << "[skip_two_backends](detail) tablet_id=" << tablet_id
+                      << ", node_id=" << node_id
+                      << ", stream_ok=" << (stream == nullptr ? "no" : "yes");
+        });
         if (stream == nullptr) {
             continue;
         }
         streams.emplace_back(std::move(stream));
     }
+    DBUG_EXECUTE_IF("VTabletWriterV2._open_streams.skip_two_backends", {
+        LOG(INFO) << "[skip_two_backends](summary) tablet_id=" << tablet_id
+                  << ", num_streams=" << streams.size()
+                  << ", num_nodes=" << location->node_ids.size();
+    });
     if (streams.size() <= location->node_ids.size() / 2) {
         return Status::InternalError("not enough streams {}/{}", streams.size(),
                                      location->node_ids.size());

From 23cddd0e24ada257cb58b52907146fef728dc723 Mon Sep 17 00:00:00 2001
From: zhiqiang <hezhiqiang@selectdb.com>
Date: Mon, 13 Jan 2025 09:48:26 +0800
Subject: [PATCH 124/140] [refactor][profile] log enhancement and minor change
 of profile (#46564)

---
 be/src/runtime/query_context.cpp              |   5 +-
 .../runtime/runtime_query_statistics_mgr.cpp  |   6 +-
 .../doris/common/proc/BackendsProcDir.java    |   2 +-
 .../common/proc/CurrentQueryInfoProvider.java |   4 +-
 .../common/{util => profile}/AggCounter.java  |   2 +-
 .../common/{util => profile}/Counter.java     |   2 +-
 .../common/profile/ExecutionProfile.java      |  12 +-
 .../apache/doris/common/profile/Profile.java  |  64 ++--
 .../doris/common/profile/ProfileManager.java  | 119 +++++---
 .../{util => profile}/RuntimeProfile.java     |   4 +-
 .../doris/common/profile/SummaryProfile.java  |   5 +-
 .../doris/load/loadv2/BrokerLoadJob.java      |   2 +-
 .../trees/plans/commands/LoadCommand.java     |   3 +-
 .../org/apache/doris/qe/QeProcessorImpl.java  |  34 ++-
 .../apache/doris/qe/QueryStatisticsItem.java  |   2 +-
 .../org/apache/doris/qe/StmtExecutor.java     |   4 +-
 .../common/profile/ProfileManagerTest.java    | 277 ++++++++++++++++++
 .../ProfilePersistentTest.java                |   8 +-
 .../{util => profile}/RuntimeProfileTest.java |   2 +-
 .../pipeline/cloud_p0/conf/be_custom.conf     |   2 +
 .../pipeline/cloud_p0/conf/fe_custom.conf     |   5 +-
 regression-test/pipeline/p0/conf/be.conf      |   2 +-
 regression-test/pipeline/p0/conf/fe.conf      |   5 +
 23 files changed, 468 insertions(+), 103 deletions(-)
 rename fe/fe-core/src/main/java/org/apache/doris/common/{util => profile}/AggCounter.java (98%)
 rename fe/fe-core/src/main/java/org/apache/doris/common/{util => profile}/Counter.java (98%)
 rename fe/fe-core/src/main/java/org/apache/doris/common/{util => profile}/RuntimeProfile.java (99%)
 create mode 100644 fe/fe-core/src/test/java/org/apache/doris/common/profile/ProfileManagerTest.java
 rename fe/fe-core/src/test/java/org/apache/doris/common/{util => profile}/ProfilePersistentTest.java (98%)
 rename fe/fe-core/src/test/java/org/apache/doris/common/{util => profile}/RuntimeProfileTest.java (99%)

diff --git a/be/src/runtime/query_context.cpp b/be/src/runtime/query_context.cpp
index c777c8100ef213c..1ff08f7d9bb8765 100644
--- a/be/src/runtime/query_context.cpp
+++ b/be/src/runtime/query_context.cpp
@@ -351,8 +351,9 @@ void QueryContext::add_fragment_profile(
 #endif
 
     std::lock_guard<std::mutex> l(_profile_mutex);
-    LOG_INFO("Query X add fragment profile, query {}, fragment {}, pipeline profile count {} ",
-             print_id(this->_query_id), fragment_id, pipeline_profiles.size());
+    VLOG_ROW << fmt::format(
+            "Query add fragment profile, query {}, fragment {}, pipeline profile count {} ",
+            print_id(this->_query_id), fragment_id, pipeline_profiles.size());
 
     _profile_map.insert(std::make_pair(fragment_id, pipeline_profiles));
 
diff --git a/be/src/runtime/runtime_query_statistics_mgr.cpp b/be/src/runtime/runtime_query_statistics_mgr.cpp
index 93d5256cad7525e..ebcaf30eab1b099 100644
--- a/be/src/runtime/runtime_query_statistics_mgr.cpp
+++ b/be/src/runtime/runtime_query_statistics_mgr.cpp
@@ -65,6 +65,8 @@ static Status _do_report_exec_stats_rpc(const TNetworkAddress& coor_addr,
         return Status::RpcError("Client rpc client failed");
     }
 
+    VLOG_DEBUG << "Sending profile";
+
     try {
         try {
             rpc_client->reportExecStatus(res, req);
@@ -272,13 +274,13 @@ void RuntimeQueryStatisticsMgr::register_fragment_profile(
 void RuntimeQueryStatisticsMgr::_report_query_profiles_function() {
     decltype(_profile_map) profile_copy;
     decltype(_load_channel_profile_map) load_channel_profile_copy;
-
+    VLOG_DEBUG << "Beging reporting profile";
     {
         std::lock_guard<std::shared_mutex> lg(_query_profile_map_lock);
         _profile_map.swap(profile_copy);
         _load_channel_profile_map.swap(load_channel_profile_copy);
     }
-
+    VLOG_DEBUG << "After swap profile map";
     // query_id -> {coordinator_addr, {fragment_id -> std::vectpr<pipeline_profile>}}
     for (auto& entry : profile_copy) {
         const auto& query_id = entry.first;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/BackendsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/BackendsProcDir.java
index c5273304137d7cf..e3db2a3a716eba7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/BackendsProcDir.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/BackendsProcDir.java
@@ -20,9 +20,9 @@
 import org.apache.doris.catalog.Env;
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.Pair;
+import org.apache.doris.common.profile.RuntimeProfile;
 import org.apache.doris.common.util.DebugUtil;
 import org.apache.doris.common.util.ListComparator;
-import org.apache.doris.common.util.RuntimeProfile;
 import org.apache.doris.common.util.TimeUtils;
 import org.apache.doris.system.Backend;
 import org.apache.doris.system.SystemInfoService;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/CurrentQueryInfoProvider.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/CurrentQueryInfoProvider.java
index 2b6d8f6702e4150..de7247ab3ab8da1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/CurrentQueryInfoProvider.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/CurrentQueryInfoProvider.java
@@ -18,9 +18,9 @@
 package org.apache.doris.common.proc;
 
 import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.util.Counter;
+import org.apache.doris.common.profile.Counter;
+import org.apache.doris.common.profile.RuntimeProfile;
 import org.apache.doris.common.util.DebugUtil;
-import org.apache.doris.common.util.RuntimeProfile;
 import org.apache.doris.qe.QueryStatisticsItem;
 import org.apache.doris.thrift.TNetworkAddress;
 import org.apache.doris.thrift.TUniqueId;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/AggCounter.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/AggCounter.java
similarity index 98%
rename from fe/fe-core/src/main/java/org/apache/doris/common/util/AggCounter.java
rename to fe/fe-core/src/main/java/org/apache/doris/common/profile/AggCounter.java
index 3f080b3752fcde4..ab3b43bd3e6c491 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/AggCounter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/AggCounter.java
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package org.apache.doris.common.util;
+package org.apache.doris.common.profile;
 
 import java.util.LinkedList;
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Counter.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Counter.java
similarity index 98%
rename from fe/fe-core/src/main/java/org/apache/doris/common/util/Counter.java
rename to fe/fe-core/src/main/java/org/apache/doris/common/profile/Counter.java
index f6c068900479d8c..f306d7c73fbb2cd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Counter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Counter.java
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package org.apache.doris.common.util;
+package org.apache.doris.common.profile;
 
 import org.apache.doris.common.io.Text;
 import org.apache.doris.persist.gson.GsonUtils;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecutionProfile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecutionProfile.java
index d2300cd667d77c5..fffcca49bf6e19e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecutionProfile.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecutionProfile.java
@@ -20,7 +20,6 @@
 import org.apache.doris.common.Pair;
 import org.apache.doris.common.Status;
 import org.apache.doris.common.util.DebugUtil;
-import org.apache.doris.common.util.RuntimeProfile;
 import org.apache.doris.planner.PlanFragmentId;
 import org.apache.doris.thrift.TDetailedReportParams;
 import org.apache.doris.thrift.TNetworkAddress;
@@ -235,7 +234,9 @@ public Status updateProfile(TQueryProfile profile, TNetworkAddress backendHBAddr
             String suffix = " (host=" + backendHBAddress + ")";
             for (TDetailedReportParams pipelineProfile : fragmentProfile) {
                 String name = "";
-                if (pipelineProfile.isSetIsFragmentLevel() && pipelineProfile.is_fragment_level) {
+                boolean isFragmentLevel = (pipelineProfile.isSetIsFragmentLevel() && pipelineProfile.is_fragment_level);
+                if (isFragmentLevel) {
+                    // Fragment Level profile is also represented by TDetailedReportParams.
                     name = "Fragment Level Profile: " + suffix;
                 } else {
                     name = "Pipeline :" + pipelineIdx + " " + suffix;
@@ -243,9 +244,9 @@ public Status updateProfile(TQueryProfile profile, TNetworkAddress backendHBAddr
                 }
 
                 RuntimeProfile profileNode = new RuntimeProfile(name);
-                // The taskprofile is used to save the profile of the pipeline, without
+                // The taskProfile is used to save the profile of the pipeline, without
                 // considering the FragmentLevel.
-                if (!(pipelineProfile.isSetIsFragmentLevel() && pipelineProfile.is_fragment_level)) {
+                if (!isFragmentLevel) {
                     taskProfile.add(profileNode);
                 }
                 if (!pipelineProfile.isSetProfile()) {
@@ -260,6 +261,9 @@ public Status updateProfile(TQueryProfile profile, TNetworkAddress backendHBAddr
             setMultiBeProfile(fragmentId, backendHBAddress, taskProfile);
         }
 
+        LOG.info("Profile update finished query: {} fragments: {} isDone: {}",
+                DebugUtil.printId(getQueryId()), profile.getFragmentIdToProfile().size(), isDone);
+
         if (profile.isSetLoadChannelProfiles()) {
             for (TRuntimeProfileTree loadChannelProfile : profile.getLoadChannelProfiles()) {
                 this.loadChannelProfile.update(loadChannelProfile);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java
index a51732474035b41..80a79ebffbf6b51 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java
@@ -20,7 +20,6 @@
 import org.apache.doris.common.Config;
 import org.apache.doris.common.io.Text;
 import org.apache.doris.common.util.DebugUtil;
-import org.apache.doris.common.util.RuntimeProfile;
 import org.apache.doris.nereids.NereidsPlanner;
 import org.apache.doris.nereids.trees.plans.AbstractPlan;
 import org.apache.doris.nereids.trees.plans.Plan;
@@ -81,10 +80,6 @@ public class Profile {
     // profile file name format: time_id
     private static final String SEPERATOR = "_";
 
-    // id will be assigned to id of SummaryProfile.
-    // For broker load, its SummaryProfile id is a string representation of a long integer,
-    // for others, it is queryID
-    private String id = "";
     // summaryProfile will be serialized to storage as JSON, and we can recover it from storage
     // recover of SummaryProfile is important, because it contains the meta information of the profile
     // we need it to construct memory index for profile retrieving.
@@ -103,7 +98,7 @@ public class Profile {
     // when coordinator finishes, it will mark finish time.
     // we will wait for about 5 seconds to see if all profiles have been reported.
     // if not, we will store the profile to storage, and release the memory,
-    // futher report will be ignored.
+    // further report will be ignored.
     // why MAX_VALUE? So that we can use PriorityQueue to sort profile by finish time decreasing order.
     private long queryFinishTimestamp = Long.MAX_VALUE;
     private Map<Integer, String> planNodeMap = Maps.newHashMap();
@@ -193,7 +188,6 @@ public static Profile read(String path) {
             DataInput dataInput = new DataInputStream(profileFileInputStream);
             Profile res = new Profile();
             res.summaryProfile = SummaryProfile.read(dataInput);
-            res.setId(res.summaryProfile.getProfileId());
             res.profileStoragePath = path;
             res.isQueryFinished = true;
             res.profileSize = fileSize;
@@ -262,7 +256,7 @@ public static String decompressExecutionProfile(byte[] data) throws IOException
     // For load task, the profile contains many execution profiles
     public void addExecutionProfile(ExecutionProfile executionProfile) {
         if (executionProfile == null) {
-            LOG.warn("try to set a null excecution profile, it is abnormal", new Exception());
+            LOG.warn("try to set a null execution profile, it is abnormal", new Exception());
             return;
         }
         executionProfile.setSummaryProfile(summaryProfile);
@@ -298,18 +292,17 @@ public synchronized void updateSummary(Map<String, String> summaryInfo, boolean
             }
 
             summaryProfile.update(summaryInfo);
-            this.setId(summaryProfile.getProfileId());
 
             if (isFinished) {
                 this.markQueryFinished(System.currentTimeMillis());
             }
-            // Nerids native insert not set planner, so it is null
+            // Nereids native insert not set planner, so it is null
             if (planner != null) {
                 this.planNodeMap = planner.getExplainStringMap();
             }
             ProfileManager.getInstance().pushProfile(this);
         } catch (Throwable t) {
-            LOG.warn("update profile {} failed", id, t);
+            LOG.warn("update profile {} failed", getId(), t);
             throw t;
         }
     }
@@ -352,7 +345,7 @@ private void waitProfileCompleteIfNeeded() {
     }
 
     private RuntimeProfile composeRootProfile() {
-        RuntimeProfile rootProfile = new RuntimeProfile(id);
+        RuntimeProfile rootProfile = new RuntimeProfile(getId());
         rootProfile.addChild(summaryProfile.getSummary());
         rootProfile.addChild(summaryProfile.getExecutionSummary());
         for (ExecutionProfile executionProfile : executionProfiles) {
@@ -378,8 +371,7 @@ public void getExecutionProfileContent(StringBuilder builder) {
             return;
         }
 
-        // Only generate merged profile for select, insert into select.
-        // Not support broker load now.
+        // For broker load, if it has more than one execution profile, we will not generate merged profile.
         RuntimeProfile mergedProfile = null;
         if (this.profileLevel == MergedProfileLevel && this.executionProfiles.size() == 1) {
             try {
@@ -389,7 +381,7 @@ public void getExecutionProfileContent(StringBuilder builder) {
                     updateActualRowCountOnPhysicalPlan(physicalPlan);
                 }
             } catch (Throwable aggProfileException) {
-                LOG.warn("build merged simple profile {} failed", this.id, aggProfileException);
+                LOG.warn("build merged simple profile {} failed", getId(), aggProfileException);
             }
         }
 
@@ -433,10 +425,6 @@ public long getQueryFinishTimestamp() {
         return this.queryFinishTimestamp;
     }
 
-    public void setId(String id) {
-        this.id = id;
-    }
-
     // For UT
     public void setSummaryProfile(SummaryProfile summaryProfile) {
         this.summaryProfile = summaryProfile;
@@ -483,7 +471,7 @@ public boolean shouldStoreToStorage() {
                     > (this.executionProfiles.size() * autoProfileDurationMs)) {
                 if (LOG.isDebugEnabled()) {
                     LOG.debug("Query/LoadJob {} costs {} ms, begin {} finish {}, need store its profile",
-                            id, durationMs, summaryProfile.getQueryBeginTime(), this.queryFinishTimestamp);
+                            getId(), durationMs, summaryProfile.getQueryBeginTime(), this.queryFinishTimestamp);
                 }
                 return true;
             }
@@ -491,7 +479,7 @@ public boolean shouldStoreToStorage() {
         }
 
         if (this.queryFinishTimestamp == Long.MAX_VALUE) {
-            LOG.warn("Logical error, query {} has finished, but queryFinishTimestamp is not set,", id);
+            LOG.warn("Logical error, query {} has finished, but queryFinishTimestamp is not set,", getId());
             return false;
         }
 
@@ -501,7 +489,8 @@ public boolean shouldStoreToStorage() {
                         > Config.profile_waiting_time_for_spill_seconds * 1000) {
             LOG.warn("Profile {} should be stored to storage without waiting for incoming profile,"
                     + " since it has been waiting for {} ms, current time {} query finished time: {}",
-                    id, currentTimeMillis - this.queryFinishTimestamp, currentTimeMillis, this.queryFinishTimestamp);
+                    getId(), currentTimeMillis - this.queryFinishTimestamp, currentTimeMillis,
+                    this.queryFinishTimestamp);
 
             this.summaryProfile.setSystemMessage(
                             "This profile is not complete, since its collection does not finish in time."
@@ -526,7 +515,7 @@ public boolean profileHasBeenStored() {
     public void markQueryFinished(long queryFinishTime) {
         try {
             if (this.profileHasBeenStored()) {
-                LOG.error("Logical error, profile {} has already been stored to storage", this.id);
+                LOG.error("Logical error, profile {} has already been stored to storage", getId());
                 return;
             }
 
@@ -539,13 +528,13 @@ public void markQueryFinished(long queryFinishTime) {
     }
 
     public void writeToStorage(String systemProfileStorageDir) {
-        if (Strings.isNullOrEmpty(id)) {
+        if (Strings.isNullOrEmpty(getId())) {
             LOG.warn("store profile failed, name is empty");
             return;
         }
 
         if (!Strings.isNullOrEmpty(profileStoragePath)) {
-            LOG.error("Logical error, profile {} has already been stored to storage", id);
+            LOG.error("Logical error, profile {} has already been stored to storage", getId());
             return;
         }
 
@@ -586,7 +575,7 @@ public void writeToStorage(String systemProfileStorageDir) {
             dataOutputStream.flush();
             this.profileSize = profileFile.length();
         } catch (Exception e) {
-            LOG.error("write {} summary profile failed", id, e);
+            LOG.error("write {} summary profile failed", getId(), e);
             return;
         } finally {
             try {
@@ -690,7 +679,7 @@ private void getOnStorageProfile(StringBuilder builder) {
             return;
         }
 
-        LOG.info("Profile {} has been stored to storage, reading it from storage", id);
+        LOG.info("Profile {} has been stored to storage, reading it from storage", getId());
 
         FileInputStream fileInputStream = null;
 
@@ -728,4 +717,25 @@ private void getOnStorageProfile(StringBuilder builder) {
 
         return;
     }
+
+    public String debugInfo() {
+        StringBuilder builder = new StringBuilder();
+        builder.append("ProfileId:").append(getId()).append("|");
+        builder.append("StoragePath:").append(profileStoragePath).append("|");
+        builder.append("StartTimeStamp:").append(summaryProfile.getQueryBeginTime()).append("|");
+        builder.append("IsFinished:").append(isQueryFinished).append("|");
+        builder.append("FinishTimestamp:").append(queryFinishTimestamp).append("|");
+        builder.append("AutoProfileDuration: ").append(autoProfileDurationMs).append("|");
+        builder.append("ExecutionProfileCnt: ").append(executionProfiles.size()).append("|");
+        builder.append("ProfileOnStorageSize:").append(profileSize);
+        return builder.toString();
+    }
+
+    public void setQueryFinishTimestamp(long l) {
+        this.queryFinishTimestamp = l;
+    }
+
+    public String getId() {
+        return summaryProfile.getProfileId();
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileManager.java
index ef9f2f7bbaf603f..f174b6b7dcc639a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileManager.java
@@ -65,7 +65,7 @@
  * All attributes can be seen from the above.
  *
  * why the element in the finished profile array is not RuntimeProfile,
- * the purpose is let coordinator can destruct earlier(the fragment profile is in Coordinator)
+ * the purpose is let coordinator can destruct earlier (the fragment profile is in Coordinator)
  *
  */
 public class ProfileManager extends MasterDaemon {
@@ -83,7 +83,7 @@ public ProfileElement(Profile profile) {
             this.profile = profile;
         }
 
-        private final Profile profile;
+        final Profile profile;
         public Map<String, String> infoStrings = Maps.newHashMap();
         public String errMsg = "";
 
@@ -655,11 +655,6 @@ private List<ProfileElement> getProfilesToBeRemoved() {
             }
         }
 
-        if (LOG.isDebugEnabled()) {
-            LOG.debug("{} profiles size on storage: {}", profileDeque.size(),
-                        DebugUtil.printByteWithUnit(totalProfileSize));
-        }
-
         final int maxSpilledProfileNum = Config.max_spilled_profile_num;
         final long spilledProfileLimitBytes = Config.spilled_profile_storage_limit_bytes;
         List<ProfileElement> queryIdToBeRemoved = Lists.newArrayList();
@@ -803,29 +798,57 @@ private void deleteBrokenProfiles() {
         }
     }
 
+    // The init value of query finish time of profile is MAX_VALUE,
+    // So a more recent query will be on the top of the heap.
+    PriorityQueue<ProfileElement> getProfileOrderByQueryFinishTimeDesc() {
+        readLock.lock();
+        try {
+            PriorityQueue<ProfileElement> queryIdDeque = new PriorityQueue<>(Comparator.comparingLong(
+                    (ProfileElement profileElement) -> profileElement.profile.getQueryFinishTimestamp()).reversed());
+
+            queryIdToProfileMap.forEach((queryId, profileElement) -> {
+                queryIdDeque.add(profileElement);
+            });
+
+            return queryIdDeque;
+        } finally {
+            readLock.unlock();
+        }
+    }
+
     // The init value of query finish time of profile is MAX_VALUE
-    // So more recent query will be on the top of heap.
-    private PriorityQueue<ProfileElement> getProfileOrderByQueryFinishTimeDesc() {
-        PriorityQueue<ProfileElement> queryIdDeque = new PriorityQueue<>(Comparator.comparingLong(
-                (ProfileElement profileElement) -> profileElement.profile.getQueryFinishTimestamp()).reversed());
+    // So query finished earlier will be on the top of heap
+    PriorityQueue<ProfileElement> getProfileOrderByQueryFinishTime() {
+        readLock.lock();
+        try {
+            PriorityQueue<ProfileElement> queryIdDeque = new PriorityQueue<>(Comparator.comparingLong(
+                    (ProfileElement profileElement) -> profileElement.profile.getQueryFinishTimestamp()));
 
-        queryIdToProfileMap.forEach((queryId, profileElement) -> {
-            queryIdDeque.add(profileElement);
-        });
+            queryIdToProfileMap.forEach((queryId, profileElement) -> {
+                queryIdDeque.add(profileElement);
+            });
 
-        return queryIdDeque;
+            return queryIdDeque;
+        } finally {
+            readLock.unlock();
+        }
     }
 
     // Older query will be on the top of heap
-    private PriorityQueue<ProfileElement> getProfileOrderByQueryStartTime() {
-        PriorityQueue<ProfileElement> queryIdDeque = new PriorityQueue<>(Comparator.comparingLong(
-                (ProfileElement profileElement) -> profileElement.profile.getSummaryProfile().getQueryBeginTime()));
+    PriorityQueue<ProfileElement> getProfileOrderByQueryStartTime() {
+        readLock.lock();
+        try {
+            PriorityQueue<ProfileElement> queryIdDeque = new PriorityQueue<>(Comparator.comparingLong(
+                    (ProfileElement profileElement) -> profileElement.profile.getSummaryProfile().getQueryBeginTime()));
 
-        queryIdToProfileMap.forEach((queryId, profileElement) -> {
-            queryIdDeque.add(profileElement);
-        });
+            queryIdToProfileMap.forEach((queryId, profileElement) -> {
+                queryIdDeque.add(profileElement);
+            });
 
-        return queryIdDeque;
+            return queryIdDeque;
+        } finally {
+            readLock.unlock();
+        }
     }
 
     // When the query is finished, the execution profile should be marked as finished
@@ -885,7 +908,7 @@ private void preventExecutionProfileLeakage() {
             writeLock.unlock();
             if (stringBuilder.length() != 0) {
                 LOG.warn("Remove expired execution profiles {}, current execution profile map size {},"
-                        + "Config.max_query_profile_num{}, Config.profile_async_collect_expire_time_secs {}",
+                        + "Config.max_query_profile_num {}, Config.profile_async_collect_expire_time_secs {}",
                         stringBuilder.toString(), executionProfileNum,
                         Config.max_query_profile_num, Config.profile_async_collect_expire_time_secs);
             }
@@ -894,7 +917,7 @@ private void preventExecutionProfileLeakage() {
 
     private void deleteOutdatedProfilesFromMemory() {
         StringBuilder stringBuilder = new StringBuilder();
-        int profileNum = 0;
+        StringBuilder stringBuilderTTL = new StringBuilder();
         writeLock.lock();
 
         try {
@@ -903,7 +926,13 @@ private void deleteOutdatedProfilesFromMemory() {
 
             for (ProfileElement profileElement : this.queryIdToProfileMap.values()) {
                 if (profileElement.profile.shouldBeRemoveFromMemory()) {
-                    profilesToRemove.add(profileElement.profile.getSummaryProfile().getProfileId());
+                    String profileId = profileElement.profile.getSummaryProfile().getProfileId();
+                    profilesToRemove.add(profileId);
+                    stringBuilder.append(profileId).append(",");
+                    if (LOG.isDebugEnabled()) {
+                        LOG.debug("Profile {} should be filtered from memory, information {}", profileId,
+                                profileElement.profile.debugInfo());
+                    }
                 }
             }
 
@@ -913,34 +942,54 @@ private void deleteOutdatedProfilesFromMemory() {
                 for (ExecutionProfile executionProfile : profileElement.profile.getExecutionProfiles()) {
                     queryIdToExecutionProfiles.remove(executionProfile.getQueryId());
                 }
-                stringBuilder.append(profileElement.profile.getSummaryProfile().getProfileId()).append(",");
             }
 
             if (this.queryIdToProfileMap.size() <= Config.max_query_profile_num) {
                 return;
             }
 
-            PriorityQueue<ProfileElement> queueIdDeque = getProfileOrderByQueryStartTime();
+            // profile is ordered by query finish time
+            // query finished earlier will be on the top of heap
+            // query finished time of unfinished query is INT_MAX, so they will be on the bottom of the heap.
+            PriorityQueue<ProfileElement> queueIdDeque = getProfileOrderByQueryFinishTime();
 
-            while (queueIdDeque.size() > Config.max_query_profile_num) {
+            while (queueIdDeque.size() > Config.max_query_profile_num && !queueIdDeque.isEmpty()) {
                 ProfileElement profileElement = queueIdDeque.poll();
-
-                queryIdToProfileMap.remove(profileElement.profile.getSummaryProfile().getProfileId());
+                String profileId = profileElement.profile.getSummaryProfile().getProfileId();
+                stringBuilderTTL.append(profileId).append(",");
+                queryIdToProfileMap.remove(profileId);
                 for (ExecutionProfile executionProfile : profileElement.profile.getExecutionProfiles()) {
                     queryIdToExecutionProfiles.remove(executionProfile.getQueryId());
                 }
 
-                stringBuilder.append(profileElement.profile.getSummaryProfile().getProfileId()).append(",");
+                if (LOG.isDebugEnabled()) {
+                    LOG.debug("Remove profile {} since ttl from memory, info {}", profileId,
+                                        profileElement.profile.debugInfo());
+                }
             }
         } finally {
-            profileNum = queryIdToProfileMap.size();
+            int profileNum = queryIdToProfileMap.size();
             writeLock.unlock();
 
-            if (stringBuilder.length() != 0) {
-                LOG.info("Remove outdated profiles {} from memoy, current profile map size {}",
-                        stringBuilder.toString(), profileNum);
+            if (stringBuilder.length() != 0 || stringBuilderTTL.length() != 0) {
+                LOG.info("Filtered profiles {}, outdated profiles {}, they are removed from memory,"
+                                + " current profile map size {}",
+                        stringBuilder.toString(), stringBuilderTTL.toString(), profileNum);
+            }
+        }
+    }
+
+    String getDebugInfo() {
+        StringBuilder stringBuilder = new StringBuilder();
+        readLock.lock();
+        try {
+            for (ProfileElement profileElement : queryIdToProfileMap.values()) {
+                stringBuilder.append(profileElement.profile.debugInfo()).append("\n");
             }
+        } finally {
+            readLock.unlock();
         }
+        return stringBuilder.toString();
     }
 
     public List<List<String>> getProfileMetaWithType(ProfileType profileType, long limit) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/RuntimeProfile.java
similarity index 99%
rename from fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java
rename to fe/fe-core/src/main/java/org/apache/doris/common/profile/RuntimeProfile.java
index 190829590341563..78ff0f58ed2f68a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/RuntimeProfile.java
@@ -15,12 +15,12 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package org.apache.doris.common.util;
+package org.apache.doris.common.profile;
 
 import org.apache.doris.common.Pair;
 import org.apache.doris.common.Reference;
 import org.apache.doris.common.io.Text;
-import org.apache.doris.common.profile.SummaryProfile;
+import org.apache.doris.common.util.DebugUtil;
 import org.apache.doris.persist.gson.GsonUtils;
 import org.apache.doris.thrift.TCounter;
 import org.apache.doris.thrift.TRuntimeProfileNode;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java
index 5b0d5ba353387fa..7215b8a9c65889c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java
@@ -19,7 +19,6 @@
 
 import org.apache.doris.common.Config;
 import org.apache.doris.common.io.Text;
-import org.apache.doris.common.util.RuntimeProfile;
 import org.apache.doris.common.util.TimeUtils;
 import org.apache.doris.persist.gson.GsonUtils;
 import org.apache.doris.thrift.TNetworkAddress;
@@ -543,8 +542,8 @@ public void setNereidsDistributeTime() {
         this.nereidsDistributeFinishTime = TimeUtils.getStartTimeMs();
     }
 
-    public void setQueryBeginTime() {
-        this.queryBeginTime = TimeUtils.getStartTimeMs();
+    public void setQueryBeginTime(long queryBeginTime) {
+        this.queryBeginTime = queryBeginTime;
     }
 
     public void setQueryAnalysisFinishTime() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
index b2fd6746f2fe692..efb2fcc06d44c6f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
@@ -247,7 +247,7 @@ private void createLoadingTask(Database db, BrokerPendingTaskAttachment attachme
                     true,
                     Integer.valueOf(sessionVariables.getOrDefault(SessionVariable.PROFILE_LEVEL, "3")),
                     Integer.valueOf(sessionVariables.getOrDefault(SessionVariable.AUTO_PROFILE_THRESHOLD_MS, "-1")));
-            this.jobProfile.getSummaryProfile().setQueryBeginTime();
+            this.jobProfile.getSummaryProfile().setQueryBeginTime(TimeUtils.getStartTimeMs());
             // TODO: 怎么给这些 load job 设置 profile 记录时间
             // this.jobProfile.setId("BrokerLoadJob " + id + ". " + label);
         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java
index aed3cb4c1f2aa06..69cbf762c2afe68 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java
@@ -29,6 +29,7 @@
 import org.apache.doris.common.profile.Profile;
 import org.apache.doris.common.util.FileFormatConstants;
 import org.apache.doris.common.util.FileFormatUtils;
+import org.apache.doris.common.util.TimeUtils;
 import org.apache.doris.datasource.property.constants.S3Properties;
 import org.apache.doris.job.base.JobExecuteType;
 import org.apache.doris.job.base.JobExecutionConfiguration;
@@ -133,7 +134,7 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
                 ctx.getSessionVariable().enableProfile,
                 ctx.getSessionVariable().profileLevel,
                 ctx.getSessionVariable().getAutoProfileThresholdMs());
-        profile.getSummaryProfile().setQueryBeginTime();
+        profile.getSummaryProfile().setQueryBeginTime(TimeUtils.getStartTimeMs());
         if (sourceInfos.size() == 1) {
             plans = ImmutableList.of(new InsertIntoTableCommand(completeQueryPlan(ctx, sourceInfos.get(0)),
                     Optional.of(labelName), Optional.empty(), Optional.empty()));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java b/fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java
index d9b4583d71f20e6..378d37d082ff9f9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java
@@ -75,18 +75,25 @@ private QeProcessorImpl() {
     private Status processQueryProfile(TQueryProfile profile, TNetworkAddress address, boolean isDone) {
         ExecutionProfile executionProfile = ProfileManager.getInstance().getExecutionProfile(profile.query_id);
         if (executionProfile == null) {
-            LOG.warn("Could not find execution profile with query id {}", DebugUtil.printId(profile.query_id));
+            LOG.warn("Could not find execution profile, query {} be {}",
+                                DebugUtil.printId(profile.query_id), address.toString());
             return new Status(TStatusCode.NOT_FOUND, "Could not find execution profile with query id "
                     + DebugUtil.printId(profile.query_id));
         }
 
-        // Update profile may cost a lot of time, use a seperate pool to deal with it.
-        writeProfileExecutor.submit(new Runnable() {
-            @Override
-            public void run() {
-                executionProfile.updateProfile(profile, address, isDone);
-            }
-        });
+        // Update profile may cost a lot of time, use a separate pool to deal with it.
+        try {
+            writeProfileExecutor.submit(new Runnable() {
+                @Override
+                public void run() {
+                    executionProfile.updateProfile(profile, address, isDone);
+                }
+            });
+        } catch (Exception e) {
+            LOG.warn("Failed to submit profile write task, query {} be {}",
+                                DebugUtil.printId(profile.query_id), address.toString());
+            return new Status(TStatusCode.INTERNAL_ERROR, "Failed to submit profile write task");
+        }
 
         return Status.OK;
     }
@@ -231,9 +238,16 @@ public TReportExecStatusResult reportExecStatus(TReportExecStatusParams params,
             // with profile in a single rpc, this will make FE ignore the exec status and may lead to bug in query
             // like insert into select.
             if (params.isSetBackendId() && params.isSetDone()) {
+                LOG.info("Receive profile {} report from {}, isDone {}, fragments {}",
+                        DebugUtil.printId(params.getQueryProfile().getQueryId()), beAddr.toString(),
+                        params.isDone(), params.getQueryProfile().fragment_id_to_profile.size());
+
                 Backend backend = Env.getCurrentSystemInfo().getBackend(params.getBackendId());
-                boolean isDone = params.isDone();
-                if (backend != null) {
+                if (backend == null) {
+                    LOG.warn("Invalid report profile req, backend {} not found, query id: {}",
+                            params.getBackendId(), DebugUtil.printId(params.getQueryProfile().getQueryId()));
+                } else {
+                    boolean isDone = params.isDone();
                     // the process status is ignored by design.
                     // actually be does not care the process status of profile on fe.
                     processQueryProfile(params.getQueryProfile(), backend.getHeartbeatAddress(), isDone);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/QueryStatisticsItem.java b/fe/fe-core/src/main/java/org/apache/doris/qe/QueryStatisticsItem.java
index 76b528464d66404..c51ff24ca14f265 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/QueryStatisticsItem.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/QueryStatisticsItem.java
@@ -17,7 +17,7 @@
 
 package org.apache.doris.qe;
 
-import org.apache.doris.common.util.RuntimeProfile;
+import org.apache.doris.common.profile.RuntimeProfile;
 import org.apache.doris.thrift.TNetworkAddress;
 import org.apache.doris.thrift.TUniqueId;
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index f693f5b82aac29a..8d712dda76a08b1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -691,7 +691,7 @@ private void executeByNereids(TUniqueId queryId) throws Exception {
         }
         context.setQueryId(queryId);
         context.setStartTime();
-        profile.getSummaryProfile().setQueryBeginTime();
+        profile.getSummaryProfile().setQueryBeginTime(TimeUtils.getStartTimeMs());
         List<List<String>> changedSessionVar = VariableMgr.dumpChangedVars(context.getSessionVariable());
         profile.setChangedSessionVar(DebugUtil.prettyPrintChangedSessionVar(changedSessionVar));
         context.setStmtId(STMT_ID_GENERATOR.incrementAndGet());
@@ -995,7 +995,7 @@ private void handleQueryWithRetry(TUniqueId queryId) throws Exception {
     public void executeByLegacy(TUniqueId queryId) throws Exception {
         context.setStartTime();
 
-        profile.getSummaryProfile().setQueryBeginTime();
+        profile.getSummaryProfile().setQueryBeginTime(TimeUtils.getStartTimeMs());
         context.setStmtId(STMT_ID_GENERATOR.incrementAndGet());
         context.setQueryId(queryId);
 
diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/profile/ProfileManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/profile/ProfileManagerTest.java
new file mode 100644
index 000000000000000..d0d17505861d831
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/profile/ProfileManagerTest.java
@@ -0,0 +1,277 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.common.profile;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.PriorityQueue;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+class ProfileManagerTest {
+    // We need a logger.
+    private static final Logger LOG = LogManager.getLogger(ProfilePersistentTest.class);
+
+    private static ProfileManager profileManager;
+
+    @BeforeAll
+    static void setUp() {
+        profileManager = new ProfileManager();
+    }
+
+    @BeforeEach
+    void cleanProfile() {
+        profileManager.cleanProfile();
+    }
+
+    @Test
+    void returnsEmptyQueueWhenNoProfiles() {
+        PriorityQueue<ProfileManager.ProfileElement> result = profileManager.getProfileOrderByQueryFinishTimeDesc();
+        Assertions.assertTrue(result.isEmpty());
+        result = profileManager.getProfileOrderByQueryFinishTime();
+        Assertions.assertTrue(result.isEmpty());
+        result = profileManager.getProfileOrderByQueryStartTime();
+        Assertions.assertTrue(result.isEmpty());
+    }
+
+    static Profile constructProfile(String id) {
+        Profile profile = new Profile();
+        SummaryProfile summaryProfile = new SummaryProfile();
+        summaryProfile.getSummary().getInfoStrings().put(SummaryProfile.PROFILE_ID, id);
+        profile.setSummaryProfile(summaryProfile);
+        return profile;
+    }
+
+    @Test
+    void getProfileByOrder() {
+        final int normalProfiles = 100;
+        for (int i = 0; i < normalProfiles; i++) {
+            Profile profile = constructProfile(String.valueOf(i));
+            Random random = new Random();
+            profile.setQueryFinishTimestamp(random.nextInt(200 - 101) + 101);
+            // set query start time in range of [0, 1000)
+            profile.getSummaryProfile().setQueryBeginTime(random.nextInt(100));
+            profileManager.pushProfile(profile);
+
+            if (i == 10) {
+                LOG.info("Profile manager debug info: {}", profileManager.getDebugInfo());
+            }
+        }
+
+        // Insert two profiles with default value.
+        Profile profile1 = constructProfile("Default 1");
+        profileManager.pushProfile(profile1);
+        Profile profile2 = constructProfile("Default 2");
+        profileManager.pushProfile(profile2);
+
+        profile1 = constructProfile("Default 3");
+        profile1.setQueryFinishTimestamp(1000L);
+        profileManager.pushProfile(profile1);
+        profile1 = constructProfile("Default 4");
+        profile1.setQueryFinishTimestamp(1000L);
+        profileManager.pushProfile(profile1);
+
+        profile1 = constructProfile("Default 5");
+        profile1.getSummaryProfile().setQueryBeginTime(1000L);
+        profileManager.pushProfile(profile1);
+        profile1 = constructProfile("Default 6");
+        profile1.getSummaryProfile().setQueryBeginTime(1000L);
+        profileManager.pushProfile(profile1);
+
+
+        Set<String> profileThatHasQueryFinishTime1000 = new HashSet<>();
+        profileThatHasQueryFinishTime1000.add("Default 3");
+        profileThatHasQueryFinishTime1000.add("Default 4");
+        Set<String> profileThatHasQueryStartTime1000 = new HashSet<>();
+        profileThatHasQueryStartTime1000.add("Default 5");
+        profileThatHasQueryStartTime1000.add("Default 6");
+        Set<String> profileThatHasDefaultQueryFinishTime = new HashSet<>();
+        profileThatHasDefaultQueryFinishTime.add("Default 1");
+        profileThatHasDefaultQueryFinishTime.add("Default 2");
+        profileThatHasDefaultQueryFinishTime.add("Default 5");
+        profileThatHasDefaultQueryFinishTime.add("Default 6");
+        Set<String> profileThatHasDefaultQueryStartTime = new HashSet<>();
+        profileThatHasDefaultQueryStartTime.add("Default 1");
+        profileThatHasDefaultQueryStartTime.add("Default 2");
+        profileThatHasDefaultQueryStartTime.add("Default 3");
+        profileThatHasDefaultQueryStartTime.add("Default 4");
+
+
+        // Profile should be ordered by query finish time in descending order.
+        // Meas that the profile with the latest query finish time should be at the top of the queue.
+        PriorityQueue<ProfileManager.ProfileElement> orderedResults = profileManager.getProfileOrderByQueryFinishTimeDesc();
+        assert orderedResults != null;
+        assert !orderedResults.isEmpty();
+        Assertions.assertEquals(106, orderedResults.size(), profileManager.getDebugInfo());
+
+        for (int i = 0; i < profileThatHasDefaultQueryFinishTime.size(); i++) {
+            ProfileManager.ProfileElement result = orderedResults.poll();
+            Assertions.assertNotEquals(result, null);
+            Assertions.assertTrue(profileThatHasDefaultQueryFinishTime.contains(result.profile.getId()));
+        }
+        for (int i = 0; i < profileThatHasQueryFinishTime1000.size(); i++) {
+            ProfileManager.ProfileElement result = orderedResults.poll();
+            Assertions.assertNotEquals(result, null);
+            Assertions.assertTrue(profileThatHasQueryFinishTime1000.contains(result.profile.getId()));
+        }
+
+        long prevQueryFinishTime = 1000L;
+        for (int i = 0; i < normalProfiles; i++) {
+            ProfileManager.ProfileElement result = orderedResults.poll();
+            Assertions.assertNotEquals(result, null);
+            Assertions.assertTrue(result.profile.getQueryFinishTimestamp() <= prevQueryFinishTime);
+            prevQueryFinishTime = result.profile.getQueryFinishTimestamp();
+        }
+
+        orderedResults = profileManager.getProfileOrderByQueryFinishTime();
+        Assertions.assertEquals(orderedResults.size(), 106);
+        // Profile should be ordered by query finish time in ascending order.
+        prevQueryFinishTime = Long.MIN_VALUE;
+        for (int i = 0; i < normalProfiles; i++) {
+            ProfileManager.ProfileElement result = orderedResults.poll();
+            Assertions.assertNotEquals(result, null);
+            Assertions.assertTrue(result.profile.getQueryFinishTimestamp() >= prevQueryFinishTime);
+            prevQueryFinishTime = result.profile.getQueryFinishTimestamp();
+        }
+        for (int i = 0; i < profileThatHasQueryFinishTime1000.size(); i++) {
+            ProfileManager.ProfileElement result = orderedResults.poll();
+            Assertions.assertNotEquals(result, null);
+            Assertions.assertTrue(profileThatHasQueryFinishTime1000.contains(result.profile.getId()));
+        }
+        for (int i = 0; i < profileThatHasDefaultQueryFinishTime.size(); i++) {
+            ProfileManager.ProfileElement result = orderedResults.poll();
+            Assertions.assertNotEquals(result, null);
+            Assertions.assertTrue(profileThatHasDefaultQueryFinishTime.contains(result.profile.getId()));
+        }
+
+        orderedResults = profileManager.getProfileOrderByQueryStartTime();
+        Assertions.assertEquals(orderedResults.size(), 106);
+        // Profile should be ordered by query start time in ascending order.
+        long prevQueryStartTime = -1;
+        for (int i = 0; i < profileThatHasDefaultQueryStartTime.size(); i++) {
+            ProfileManager.ProfileElement result = orderedResults.poll();
+            Assertions.assertNotEquals(result, null);
+            Assertions.assertTrue(profileThatHasDefaultQueryStartTime.contains(result.profile.getId()),
+                                result.profile.getId() + " " + result.profile.getSummaryProfile().getQueryBeginTime());
+        }
+
+        for (int i = 0; i < normalProfiles; i++) {
+            ProfileManager.ProfileElement result = orderedResults.poll();
+            Assertions.assertNotEquals(result, null);
+            Assertions.assertTrue(result.profile.getSummaryProfile().getQueryBeginTime() >= prevQueryStartTime);
+            prevQueryStartTime = result.profile.getSummaryProfile().getQueryBeginTime();
+        }
+
+        for (int i = 0; i < profileThatHasQueryStartTime1000.size(); i++) {
+            ProfileManager.ProfileElement result = orderedResults.poll();
+            Assertions.assertNotEquals(result, null);
+            Assertions.assertTrue(profileThatHasQueryStartTime1000.contains(result.profile.getId()),
+                                result.profile.getId() + " " + result.profile.getSummaryProfile().getQueryBeginTime());
+        }
+    }
+
+    @Test
+    void getProfileByOrderParallel() throws InterruptedException {
+        // Test the parallel case.
+        // Create a thread pool with 3 threads.
+        final int threadNum = 3;
+        List<Thread> threads = new ArrayList<>();
+        AtomicBoolean stopFlag = new AtomicBoolean(false);
+
+        // These threads keep adding profiles to the profile manager.
+        // The profile they create has random name, random query finish time and random query start time.
+        for (int i = 0; i < threadNum; i++) {
+            threads.add(new Thread(() -> {
+                Random random = new Random();
+                for (int j = 0; j < 100; j++) {
+                    Profile profile = constructProfile(String.valueOf(random.nextInt(1000)));
+                    profile.getSummaryProfile().setQueryBeginTime(random.nextInt(1000));
+                    profile.setQueryFinishTimestamp(random.nextInt(2000) + 1000);
+                    profileManager.pushProfile(profile);
+                }
+            }));
+        }
+        // Create another thread to get the profile by different order.
+        for (int i = 0; i < threadNum; i++) {
+            threads.add(new Thread(() -> {
+                while (!stopFlag.get()) {
+                    PriorityQueue<ProfileManager.ProfileElement> orderedResults = profileManager.getProfileOrderByQueryFinishTimeDesc();
+                    long prevQueryFinishTime = Long.MAX_VALUE;
+                    while (!orderedResults.isEmpty()) {
+                        ProfileManager.ProfileElement result = orderedResults.poll();
+                        Assertions.assertTrue(result.profile.getQueryFinishTimestamp() <= prevQueryFinishTime);
+                        prevQueryFinishTime = result.profile.getQueryFinishTimestamp();
+                    }
+                }
+            }));
+        }
+
+        for (int i = 0; i < threadNum; i++) {
+            threads.add(new Thread(() -> {
+                while (!stopFlag.get()) {
+                    PriorityQueue<ProfileManager.ProfileElement> orderedResults = profileManager.getProfileOrderByQueryStartTime();
+                    long prevQueryStartTime = -1;
+                    while (!orderedResults.isEmpty()) {
+                        ProfileManager.ProfileElement result = orderedResults.poll();
+                        Assertions.assertTrue(result.profile.getSummaryProfile().getQueryBeginTime() >= prevQueryStartTime);
+                        prevQueryStartTime = result.profile.getSummaryProfile().getQueryBeginTime();
+                    }
+                }
+            }));
+        }
+
+        for (int i = 0; i < threadNum; i++) {
+            threads.add(new Thread(() -> {
+                while (!stopFlag.get()) {
+                    PriorityQueue<ProfileManager.ProfileElement> orderedResults = profileManager.getProfileOrderByQueryFinishTime();
+                    long prevQueryFinishTime = Long.MIN_VALUE;
+                    while (!orderedResults.isEmpty()) {
+                        ProfileManager.ProfileElement result = orderedResults.poll();
+                        Assertions.assertTrue(result.profile.getQueryFinishTimestamp() >= prevQueryFinishTime);
+                        prevQueryFinishTime = result.profile.getQueryFinishTimestamp();
+                    }
+                }
+            }));
+        }
+
+        for (Thread thread : threads) {
+            thread.start();
+        }
+
+        Thread.sleep(5000);
+
+        stopFlag.set(true);
+
+        for (Thread thread : threads) {
+            try {
+                thread.join();
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+        }
+    }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/ProfilePersistentTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/profile/ProfilePersistentTest.java
similarity index 98%
rename from fe/fe-core/src/test/java/org/apache/doris/common/util/ProfilePersistentTest.java
rename to fe/fe-core/src/test/java/org/apache/doris/common/profile/ProfilePersistentTest.java
index c21c57ba1d480ad..cb1804e2d500853 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/common/util/ProfilePersistentTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/profile/ProfilePersistentTest.java
@@ -15,12 +15,11 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package org.apache.doris.common.util;
+package org.apache.doris.common.profile;
 
-import org.apache.doris.common.profile.ExecutionProfile;
-import org.apache.doris.common.profile.Profile;
-import org.apache.doris.common.profile.SummaryProfile;
 import org.apache.doris.common.profile.SummaryProfile.SummaryBuilder;
+import org.apache.doris.common.util.DebugUtil;
+import org.apache.doris.common.util.TimeUtils;
 import org.apache.doris.thrift.QueryState;
 import org.apache.doris.thrift.TUniqueId;
 import org.apache.doris.thrift.TUnit;
@@ -77,7 +76,6 @@ public static Profile constructRandomProfile(int executionProfileNum) {
         SummaryProfile summaryProfile = constructRandomSummaryProfile();
         String stringUniqueId = summaryProfile.getProfileId();
         TUniqueId thriftUniqueId = DebugUtil.parseTUniqueIdFromString(stringUniqueId);
-        profile.setId(stringUniqueId);
         profile.setSummaryProfile(summaryProfile);
 
         for (int i = 0; i < executionProfileNum; i++) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/RuntimeProfileTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/profile/RuntimeProfileTest.java
similarity index 99%
rename from fe/fe-core/src/test/java/org/apache/doris/common/util/RuntimeProfileTest.java
rename to fe/fe-core/src/test/java/org/apache/doris/common/profile/RuntimeProfileTest.java
index 56ed66c050407f1..93cb5e357941485 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/common/util/RuntimeProfileTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/profile/RuntimeProfileTest.java
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package org.apache.doris.common.util;
+package org.apache.doris.common.profile;
 
 import org.apache.doris.thrift.TCounter;
 import org.apache.doris.thrift.TRuntimeProfileNode;
diff --git a/regression-test/pipeline/cloud_p0/conf/be_custom.conf b/regression-test/pipeline/cloud_p0/conf/be_custom.conf
index d201cad3facc27e..c35859cd6b4f68b 100644
--- a/regression-test/pipeline/cloud_p0/conf/be_custom.conf
+++ b/regression-test/pipeline/cloud_p0/conf/be_custom.conf
@@ -39,3 +39,5 @@ pipeline_task_leakage_detect_period_sec=1
 crash_in_memory_tracker_inaccurate = true
 enable_table_size_correctness_check=true
 enable_brpc_connection_check=true
+
+sys_log_verbose_modules=query_context,runtime_query_statistics_mgr
\ No newline at end of file
diff --git a/regression-test/pipeline/cloud_p0/conf/fe_custom.conf b/regression-test/pipeline/cloud_p0/conf/fe_custom.conf
index 955409d9bb239bc..40d88d0f2076123 100644
--- a/regression-test/pipeline/cloud_p0/conf/fe_custom.conf
+++ b/regression-test/pipeline/cloud_p0/conf/fe_custom.conf
@@ -24,7 +24,10 @@ enable_debug_points = true
 disable_datev1=false
 
 disable_decimalv2=false
-max_query_profile_num=1000
+sys_log_verbose_modules = org.apache.doris.common.profile,org.apache.doris.qe.QeProcessorImpl
+# profile related
+max_query_profile_num = 2000
+max_spilled_profile_num = 2000
 
 statistics_sql_mem_limit_in_bytes=21474836480
 cpu_resource_limit_per_analyze_task=-1
diff --git a/regression-test/pipeline/p0/conf/be.conf b/regression-test/pipeline/p0/conf/be.conf
index 0b73375b3fbdd30..be798738e68a74f 100644
--- a/regression-test/pipeline/p0/conf/be.conf
+++ b/regression-test/pipeline/p0/conf/be.conf
@@ -35,7 +35,7 @@ JEMALLOC_PROF_PRFIX="jemalloc_heap_profile_"
 
 # INFO, WARNING, ERROR, FATAL
 sys_log_level = INFO
-
+sys_log_verbose_modules=query_context,runtime_query_statistics_mgr
 be_port = 9161
 webserver_port = 8141
 heartbeat_service_port = 9151
diff --git a/regression-test/pipeline/p0/conf/fe.conf b/regression-test/pipeline/p0/conf/fe.conf
index 44688ff4adc2db8..2f493ff1098cb6b 100644
--- a/regression-test/pipeline/p0/conf/fe.conf
+++ b/regression-test/pipeline/p0/conf/fe.conf
@@ -34,6 +34,7 @@ JAVA_OPTS_FOR_JDK_17="-Djavax.security.auth.useSubjectCredsOnly=false -Xmx8192m
 
 sys_log_level = INFO
 sys_log_mode = NORMAL
+sys_log_verbose_modules = org.apache.doris.common.profile,org.apache.doris.qe.QeProcessorImpl
 arrow_flight_sql_port = 8081
 catalog_trash_expire_second=1
 #enable ssl for test
@@ -85,3 +86,7 @@ enable_deadlock_detection = true
 max_lock_hold_threshold_seconds = 10
 
 force_olap_table_replication_allocation=tag.location.default:1
+
+# profile related
+max_query_profile_num = 2000
+max_spilled_profile_num = 2000

From 7ae1a2353da321108af224106a737dbd1b63c2ed Mon Sep 17 00:00:00 2001
From: "Mingyu Chen (Rayner)" <morningman@163.com>
Date: Mon, 13 Jan 2025 09:54:26 +0800
Subject: [PATCH 125/140] [fix](test) fix unstable test after some test
 migration (#46834)

### What problem does this PR solve?

fix some unstable test cases
---
 .../jdbc/test_jdbc_query_tvf.out              | 72 +++++++++----------
 ...est_switch_catalog_and_delete_internal.out |  5 ++
 .../suites/auth_p0/test_query_tvf_auth.groovy |  6 +-
 ..._switch_catalog_and_delete_internal.groovy | 10 +--
 4 files changed, 49 insertions(+), 44 deletions(-)

diff --git a/regression-test/data/external_table_p0/jdbc/test_jdbc_query_tvf.out b/regression-test/data/external_table_p0/jdbc/test_jdbc_query_tvf.out
index 3d71bfb26415ff5..b96eed60fe0d90b 100644
--- a/regression-test/data/external_table_p0/jdbc/test_jdbc_query_tvf.out
+++ b/regression-test/data/external_table_p0/jdbc/test_jdbc_query_tvf.out
@@ -1,43 +1,43 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !sql --
-bigint	BIGINT	Yes	true	\N	NONE
-bigint_u	LARGEINT	Yes	true	\N	NONE
-binary	TEXT	Yes	true	\N	NONE
-bit	TEXT	Yes	true	\N	NONE
-blob	TEXT	Yes	true	\N	NONE
-boolean	TINYINT	Yes	true	\N	NONE
-char	CHAR(6)	Yes	true	\N	NONE
-date	DATE	Yes	true	\N	NONE
-datetime	DATETIME	Yes	true	\N	NONE
-decimal	DECIMAL(12, 4)	Yes	true	\N	NONE
-decimal_u	DECIMAL(19, 5)	Yes	true	\N	NONE
-double	DOUBLE	Yes	true	\N	NONE
-double_u	DOUBLE	Yes	true	\N	NONE
-enum	CHAR(6)	Yes	true	\N	NONE
-float	FLOAT	Yes	true	\N	NONE
-float_u	FLOAT	Yes	true	\N	NONE
-int	INT	Yes	true	\N	NONE
-int_u	BIGINT	Yes	true	\N	NONE
-json	TEXT	Yes	true	\N	NONE
-mediumint	INT	Yes	true	\N	NONE
-mediumint_u	INT	Yes	true	\N	NONE
-set	CHAR(6)	Yes	true	\N	NONE
-smallint	SMALLINT	Yes	true	\N	NONE
-smallint_u	INT	Yes	true	\N	NONE
-text	TEXT	Yes	true	\N	NONE
-time	TEXT	Yes	true	\N	NONE
-timestamp	DATETIME(4)	Yes	true	\N	NONE
-tinyint	TINYINT	Yes	true	\N	NONE
-tinyint_u	SMALLINT	Yes	true	\N	NONE
-varbinary	TEXT	Yes	true	\N	NONE
-varchar	VARCHAR(10)	Yes	true	\N	NONE
-year	SMALLINT	Yes	true	\N	NONE
+bigint	bigint	Yes	true	\N	NONE
+bigint_u	largeint	Yes	true	\N	NONE
+binary	text	Yes	true	\N	NONE
+bit	text	Yes	true	\N	NONE
+blob	text	Yes	true	\N	NONE
+boolean	tinyint	Yes	true	\N	NONE
+char	char(5)	Yes	true	\N	NONE
+date	date	Yes	true	\N	NONE
+datetime	datetime	Yes	true	\N	NONE
+decimal	decimal(12,4)	Yes	true	\N	NONE
+decimal_u	decimal(19,5)	Yes	true	\N	NONE
+double	double	Yes	true	\N	NONE
+double_u	double	Yes	true	\N	NONE
+enum	char(6)	Yes	true	\N	NONE
+float	float	Yes	true	\N	NONE
+float_u	float	Yes	true	\N	NONE
+int	int	Yes	true	\N	NONE
+int_u	bigint	Yes	true	\N	NONE
+json	text	Yes	true	\N	NONE
+mediumint	int	Yes	true	\N	NONE
+mediumint_u	int	Yes	true	\N	NONE
+set	char(23)	Yes	true	\N	NONE
+smallint	smallint	Yes	true	\N	NONE
+smallint_u	int	Yes	true	\N	NONE
+text	text	Yes	true	\N	NONE
+time	text	Yes	true	\N	NONE
+timestamp	datetime(4)	Yes	true	\N	NONE
+tinyint	tinyint	Yes	true	\N	NONE
+tinyint_u	smallint	Yes	true	\N	NONE
+varbinary	text	Yes	true	\N	NONE
+varchar	varchar(10)	Yes	true	\N	NONE
+year	smallint	Yes	true	\N	NONE
 
 -- !sql --
-\N	302	\N	502	602	4.14159	\N	6.14159	\N	-124	-302	2013	-402	-502	-602	\N	2012-10-26T02:08:39.345700	2013-10-26T08:09:18	-5.14145	\N	-7.1400	row2	\N	09:11:09.567	text2	0xE86F6C6C6F20576F726C67	\N	\N	0x2F	\N	0x88656C6C9F	Value3
-201	301	401	501	601	3.14159	4.1415926	5.14159	1	-123	-301	2012	-401	-501	-601	2012-10-30	2012-10-25T12:05:36.345700	2012-10-25T08:08:08	-4.14145	-5.1400000001	-6.1400	row1	line1	09:09:09.567	text1	0x48656C6C6F20576F726C64	{"age":30, "city":"London", "name":"Alice"}	Option1,Option3	0x2A	0x48656C6C6F00000000000000	0x48656C6C6F	Value2
-202	302	402	502	602	4.14159	5.1415926	6.14159	0	-124	-302	2013	-402	-502	-602	2012-11-01	2012-10-26T02:08:39.345700	2013-10-26T08:09:18	-5.14145	-6.1400000001	-7.1400	row2	line2	09:11:09.567	text2	0xE86F6C6C6F20576F726C67	{"age":18, "city":"ChongQing", "name":"Gaoxin"}	Option1,Option2	0x2F	0x58676C6C6F00000000000000	0x88656C6C9F	Value3
-203	303	403	503	603	7.14159	8.1415926	9.14159	0	\N	-402	2017	-602	-902	-1102	2012-11-02	\N	2013-10-27T08:11:18	-5.14145	-6.1400000000001	-7.1400	row3	line3	09:11:09.567	text3	0xE86F6C6C6F20576F726C67	{"age":24, "city":"ChongQing", "name":"ChenQi"}	Option2	0x2F	0x58676C6C6F00000000000000	\N	Value1
+\N	302	\N	502	602	4.14159	\N	6.14159	\N	-124	-302	2013	-402	-502	-602	\N	2012-10-26T02:08:39.345700	2013-10-26T08:09:18	-5.14145	\N	-7.1400	row2	\N	09:11:09.5678	text2	0xE86F6C6C6F20576F726C67	\N	\N	0x2F	\N	0x88656C6C9F	Value3
+201	301	401	501	601	3.14159	4.1415926	5.14159	1	-123	-301	2012	-401	-501	-601	2012-10-30	2012-10-25T12:05:36.345700	2012-10-25T08:08:08	-4.14145	-5.1400000001	-6.1400	row1	line1	09:09:09.5678	text1	0x48656C6C6F20576F726C64	{"age": 30, "city": "London", "name": "Alice"}	Option1,Option3	0x2A	0x48656C6C6F00000000000000	0x48656C6C6F	Value2
+202	302	402	502	602	4.14159	5.1415926	6.14159	0	-124	-302	2013	-402	-502	-602	2012-11-01	2012-10-26T02:08:39.345700	2013-10-26T08:09:18	-5.14145	-6.1400000001	-7.1400	row2	line2	09:11:09.5678	text2	0xE86F6C6C6F20576F726C67	{"age": 18, "city": "ChongQing", "name": "Gaoxin"}	Option1,Option2	0x2F	0x58676C6C6F00000000000000	0x88656C6C9F	Value3
+203	303	403	503	603	7.14159	8.1415926	9.14159	0	\N	-402	2017	-602	-902	-1102	2012-11-02	\N	2013-10-27T08:11:18	-5.14145	-6.1400000000001	-7.1400	row3	line3	09:11:09.5678	text3	0xE86F6C6C6F20576F726C67	{"age": 24, "city": "ChongQing", "name": "ChenQi"}	Option2	0x2F	0x58676C6C6F00000000000000	\N	Value1
 
 -- !sql --
 4
diff --git a/regression-test/data/external_table_p0/jdbc/test_switch_catalog_and_delete_internal.out b/regression-test/data/external_table_p0/jdbc/test_switch_catalog_and_delete_internal.out
index cda9c9293b37168..c1f65cf759946b3 100644
--- a/regression-test/data/external_table_p0/jdbc/test_switch_catalog_and_delete_internal.out
+++ b/regression-test/data/external_table_p0/jdbc/test_switch_catalog_and_delete_internal.out
@@ -1,2 +1,7 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !test --
+2	1	3
+3	5	6
+4	5	6
+6	\N	6
+
diff --git a/regression-test/suites/auth_p0/test_query_tvf_auth.groovy b/regression-test/suites/auth_p0/test_query_tvf_auth.groovy
index 05c274077d9eb3f..746eb47ce5b870e 100644
--- a/regression-test/suites/auth_p0/test_query_tvf_auth.groovy
+++ b/regression-test/suites/auth_p0/test_query_tvf_auth.groovy
@@ -15,8 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_jdbc_query_tvf","p0,auth") {
-    String suiteName = "test_jdbc_query_tvf"
+suite("test_query_tvf_auth", "p0,auth,external,external_docker") {
+    String suiteName = "test_query_tvf_auth"
     String enabled = context.config.otherConfigs.get("enableJdbcTest")
     String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
     String s3_endpoint = getS3Endpoint()
@@ -58,7 +58,7 @@ suite("test_jdbc_query_tvf","p0,auth") {
                   sql """
                      select * from query('catalog' = '${catalog_name}', 'query' = 'select * from doris_test.all_types');
                   """
-                  exception "denied"
+                  exception "has no privilege"
             }
         }
         sql """grant select_priv on ${catalog_name}.*.* to ${dorisuser}"""
diff --git a/regression-test/suites/external_table_p0/jdbc/test_switch_catalog_and_delete_internal.groovy b/regression-test/suites/external_table_p0/jdbc/test_switch_catalog_and_delete_internal.groovy
index adfb20e355d258b..dd8fddf65e59aa4 100644
--- a/regression-test/suites/external_table_p0/jdbc/test_switch_catalog_and_delete_internal.groovy
+++ b/regression-test/suites/external_table_p0/jdbc/test_switch_catalog_and_delete_internal.groovy
@@ -15,16 +15,16 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_switch_catalog_and_delete_internal", "p0,external") {
+suite("test_switch_catalog_and_delete_internal", "p0,external,external_docker") {
     String enabled = context.config.otherConfigs.get("enableJdbcTest")
     String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
     String mysql_port = context.config.otherConfigs.get("mysql_57_port");
     String s3_endpoint = getS3Endpoint()
     String bucket = getS3BucketName()
     String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-java-8.0.25.jar"
+    String db = context.config.getDbNameByFile(context.file)
     if (enabled != null && enabled.equalsIgnoreCase("true")) {
         // 0.create internal db and table
-        String db = context.config.getDbNameByFile(new File(context.file))
         sql "drop table if exists test_switch_catalog_and_delete_internal"
         sql """
         create table test_switch_catalog_and_delete_internal(pk int, a int, b int) distributed by hash(pk) buckets 10
@@ -50,9 +50,9 @@ suite("test_switch_catalog_and_delete_internal", "p0,external") {
         sql "switch test_switch_catalog_and_delete_internal_catalog"
         sql "refresh catalog test_switch_catalog_and_delete_internal_catalog"
         // 3.delete table
-        sql "delete from internal.${db}.test_switch_catalog_and_delete_internal;"
+        sql "delete from internal.${db}.test_switch_catalog_and_delete_internal where pk < 2;"
         // 4.select table
-        qt_test "select * from internal.maldb.test_switch_catalog_and_delete_internal;"
+        qt_test "select * from internal.${db}.test_switch_catalog_and_delete_internal order by pk;"
     }
 
-}
\ No newline at end of file
+}

From 27bc0bd379d7aa737dbb9289cdc865fafdbc5c80 Mon Sep 17 00:00:00 2001
From: zy-kkk <zhongyk10@gmail.com>
Date: Mon, 13 Jan 2025 09:55:04 +0800
Subject: [PATCH 126/140] [fix](test) Add cloud mode computing permissions to
 the lower case test user (#46765)

---
 .../lower_case/test_conflict_name.groovy                 | 9 +++++++++
 .../lower_case/test_lower_case_meta_include.groovy       | 9 +++++++++
 .../test_lower_case_meta_show_and_select.groovy          | 9 +++++++++
 ...ase_meta_with_lower_table_conf_show_and_select.groovy | 9 +++++++++
 .../lower_case/test_lower_case_mtmv.groovy               | 9 +++++++++
 .../test_meta_cache_select_without_refresh.groovy        | 9 +++++++++
 .../lower_case/test_meta_names_mapping.groovy            | 9 +++++++++
 .../lower_case/test_timing_refresh_catalog.groovy        | 9 +++++++++
 .../external_table_p0/lower_case/upgrade/load.groovy     | 9 +++++++++
 9 files changed, 81 insertions(+)

diff --git a/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy b/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy
index 43d6a68c1ad6725..cbe2ea96ae8e5d3 100644
--- a/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy
+++ b/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy
@@ -26,6 +26,15 @@ suite("test_conflict_name", "p0,external,doris,meta_names_mapping") {
 
     try_sql """drop user ${jdbcUser}"""
     sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+
+    //cloud-mode
+    if (isCloudMode()) {
+        def clusters = sql " SHOW CLUSTERS; "
+        assertTrue(!clusters.isEmpty())
+        def validCluster = clusters[0][0]
+        sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${jdbcUser}""";
+    }
+
     sql """grant all on *.*.* to ${jdbcUser}"""
 
     sql """drop database if exists internal.external_conflict_name; """
diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy
index cd48678f25cf580..63f18e358c71bf6 100644
--- a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy
+++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy
@@ -26,6 +26,15 @@ suite("test_lower_case_meta_include", "p0,external,doris,external_docker,externa
 
     try_sql """drop user ${jdbcUser}"""
     sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+
+    //cloud-mode
+    if (isCloudMode()) {
+        def clusters = sql " SHOW CLUSTERS; "
+        assertTrue(!clusters.isEmpty())
+        def validCluster = clusters[0][0]
+        sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${jdbcUser}""";
+    }
+
     sql """grant all on *.*.* to ${jdbcUser}"""
 
     String mapping_db = """
diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy
index ac86623243d9dcd..72e945ea8ffd59a 100644
--- a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy
+++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy
@@ -42,6 +42,15 @@ suite("test_lower_case_meta_show_and_select", "p0,external,doris,external_docker
 
     try_sql """drop user ${jdbcUser}"""
     sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+
+    //cloud-mode
+    if (isCloudMode()) {
+        def clusters = sql " SHOW CLUSTERS; "
+        assertTrue(!clusters.isEmpty())
+        def validCluster = clusters[0][0]
+        sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${jdbcUser}""";
+    }
+
     sql """grant all on *.*.* to ${jdbcUser}"""
 
     sql """drop database if exists internal.external_test_lower; """
diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy
index 0c2712e46866028..42c6fd08203e4f6 100644
--- a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy
+++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy
@@ -42,6 +42,15 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external
 
     try_sql """drop user ${jdbcUser}"""
     sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+
+    //cloud-mode
+    if (isCloudMode()) {
+        def clusters = sql " SHOW CLUSTERS; "
+        assertTrue(!clusters.isEmpty())
+        def validCluster = clusters[0][0]
+        sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${jdbcUser}""";
+    }
+
     sql """grant all on *.*.* to ${jdbcUser}"""
 
     sql """drop database if exists internal.external_test_lower_with_conf; """
diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy
index f72b3c9e68da900..4cc8593e459a6c1 100644
--- a/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy
+++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy
@@ -26,6 +26,15 @@ suite("test_lower_case_mtmv", "p0,external,doris,external_docker,external_docker
 
     try_sql """drop user ${jdbcUser}"""
     sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+
+    //cloud-mode
+    if (isCloudMode()) {
+        def clusters = sql " SHOW CLUSTERS; "
+        assertTrue(!clusters.isEmpty())
+        def validCluster = clusters[0][0]
+        sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${jdbcUser}""";
+    }
+
     sql """grant all on *.*.* to ${jdbcUser}"""
 
     sql """drop database if exists internal.EXTERNAL_LOWER_MTMV; """
diff --git a/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy b/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy
index 17f37b2c13248f9..b677133015c996e 100644
--- a/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy
+++ b/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy
@@ -26,6 +26,15 @@ suite("test_meta_cache_select_without_refresh", "p0,external,doris,external_dock
 
     try_sql """drop user ${jdbcUser}"""
     sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+
+    //cloud-mode
+    if (isCloudMode()) {
+        def clusters = sql " SHOW CLUSTERS; "
+        assertTrue(!clusters.isEmpty())
+        def validCluster = clusters[0][0]
+        sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${jdbcUser}""";
+    }
+
     sql """grant all on *.*.* to ${jdbcUser}"""
 
     sql """ drop database if exists internal.external_lower_select_without_refresh; """
diff --git a/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy b/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy
index b58077295b3a7eb..2187cbad59241f9 100644
--- a/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy
+++ b/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy
@@ -26,6 +26,15 @@ suite("test_meta_names_mapping", "p0,external,doris,meta_names_mapping") {
 
     try_sql """drop user ${jdbcUser}"""
     sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+
+    //cloud-mode
+    if (isCloudMode()) {
+        def clusters = sql " SHOW CLUSTERS; "
+        assertTrue(!clusters.isEmpty())
+        def validCluster = clusters[0][0]
+        sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${jdbcUser}""";
+    }
+
     sql """grant all on *.*.* to ${jdbcUser}"""
 
     String validMetaNamesMapping = """
diff --git a/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy b/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy
index 242ca1b8a53f344..5e0386330e3a515 100644
--- a/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy
+++ b/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy
@@ -47,6 +47,15 @@ suite("test_timing_refresh_catalog", "p0,external,doris,external_docker,external
 
     try_sql """drop user ${jdbcUser}"""
     sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+
+    //cloud-mode
+    if (isCloudMode()) {
+        def clusters = sql " SHOW CLUSTERS; "
+        assertTrue(!clusters.isEmpty())
+        def validCluster = clusters[0][0]
+        sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${jdbcUser}""";
+    }
+
     sql """grant all on *.*.* to ${jdbcUser}"""
 
     String mapping = """
diff --git a/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy b/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy
index 782f7b3711904c9..053fad17785a2f2 100644
--- a/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy
+++ b/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy
@@ -42,6 +42,15 @@ suite("test_upgrade_lower_case_catalog_prepare", "p0,external,doris,external_doc
 
     try_sql """drop user ${jdbcUser}"""
     sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+
+    //cloud-mode
+    if (isCloudMode()) {
+        def clusters = sql " SHOW CLUSTERS; "
+        assertTrue(!clusters.isEmpty())
+        def validCluster = clusters[0][0]
+        sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${jdbcUser}""";
+    }
+
     sql """grant all on *.*.* to ${jdbcUser}"""
 
     sql """drop database if exists internal.upgrade_lower_case_catalog_lower; """

From d884c14f4b3fb18ed45828459b4caa5dbb3d2624 Mon Sep 17 00:00:00 2001
From: zhangstar333 <zhangsida@selectdb.com>
Date: Mon, 13 Jan 2025 10:00:58 +0800
Subject: [PATCH 127/140] [fix](test) fix set_and_unset_variable_command test
 case failed (#46678)

### What problem does this PR solve?
Problem Summary:

in this PR https://github.com/apache/doris/pull/45585
ignore the session variable enable_local_exchange
so it's couldn't get result use show variables command.
change to another session to test command
---
 .../set_and_unset_variable_command.out        | 12 +++++------
 .../set_and_unset_variable_command.groovy     | 20 +++++++++----------
 2 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/regression-test/data/nereids_p0/ddl/set_and_unset_commands/set_and_unset_variable_command.out b/regression-test/data/nereids_p0/ddl/set_and_unset_commands/set_and_unset_variable_command.out
index e7a063c5116fd0c..8c890f69c3426cc 100644
--- a/regression-test/data/nereids_p0/ddl/set_and_unset_commands/set_and_unset_variable_command.out
+++ b/regression-test/data/nereids_p0/ddl/set_and_unset_commands/set_and_unset_variable_command.out
@@ -54,16 +54,16 @@ experimental_enable_agg_state	false	false	0
 experimental_enable_agg_state	false	false	0
 
 -- !cmd --
-deprecated_enable_local_exchange	false	true	1
+deprecated_group_by_and_having_use_alias_first	false	false	0
 
 -- !cmd --
-deprecated_enable_local_exchange	true	true	0
+deprecated_group_by_and_having_use_alias_first	false	false	0
 
 -- !cmd --
-deprecated_enable_local_exchange	true	true	0
+deprecated_group_by_and_having_use_alias_first	false	false	0
 
 -- !cmd --
-deprecated_enable_local_exchange	true	true	0
+deprecated_group_by_and_having_use_alias_first	false	false	0
 
 -- !cmd --
 runtime_filter_type	IN_OR_BLOOM_FILTER,MIN_MAX	IN_OR_BLOOM_FILTER,MIN_MAX	0
@@ -72,7 +72,7 @@ runtime_filter_type	IN_OR_BLOOM_FILTER,MIN_MAX	IN_OR_BLOOM_FILTER,MIN_MAX	0
 experimental_enable_agg_state	false	false	0
 
 -- !cmd --
-deprecated_enable_local_exchange	true	true	0
+deprecated_group_by_and_having_use_alias_first	false	false	0
 
 -- !cmd --
 show_hidden_columns	false	false	0
@@ -87,7 +87,7 @@ runtime_filter_type	IN_OR_BLOOM_FILTER,MIN_MAX	IN_OR_BLOOM_FILTER,MIN_MAX	0
 experimental_enable_agg_state	false	false	0
 
 -- !cmd --
-deprecated_enable_local_exchange	true	true	0
+deprecated_group_by_and_having_use_alias_first	false	false	0
 
 -- !cmd --
 show_hidden_columns	false	false	0
diff --git a/regression-test/suites/nereids_p0/ddl/set_and_unset_commands/set_and_unset_variable_command.groovy b/regression-test/suites/nereids_p0/ddl/set_and_unset_commands/set_and_unset_variable_command.groovy
index 46e253cd085b8f1..f736b33d08a739c 100644
--- a/regression-test/suites/nereids_p0/ddl/set_and_unset_commands/set_and_unset_variable_command.groovy
+++ b/regression-test/suites/nereids_p0/ddl/set_and_unset_commands/set_and_unset_variable_command.groovy
@@ -59,22 +59,22 @@ suite("set_and_unset_variable_command", "nonConcurrent") {
     qt_cmd """show global variables like 'experimental_enable_agg_state'"""
 
     // test variables with deprecated_ prefix
-    checkNereidsExecute("set deprecated_enable_local_exchange = false")
-    qt_cmd """show session variables like 'deprecated_enable_local_exchange'"""
-    qt_cmd """show global variables like 'deprecated_enable_local_exchange'"""
-    checkNereidsExecute("UNSET global VARIABLE deprecated_enable_local_exchange")
-    qt_cmd """show session variables like 'deprecated_enable_local_exchange'"""
-    qt_cmd """show global variables like 'deprecated_enable_local_exchange'"""
+    checkNereidsExecute("set deprecated_group_by_and_having_use_alias_first = false")
+    qt_cmd """show session variables like 'deprecated_group_by_and_having_use_alias_first'"""
+    qt_cmd """show global variables like 'deprecated_group_by_and_having_use_alias_first'"""
+    checkNereidsExecute("UNSET global VARIABLE deprecated_group_by_and_having_use_alias_first")
+    qt_cmd """show session variables like 'deprecated_group_by_and_having_use_alias_first'"""
+    qt_cmd """show global variables like 'deprecated_group_by_and_having_use_alias_first'"""
 
     // test UNSET VARIABLE ALL
     checkNereidsExecute("set runtime_filter_type='BLOOM_FILTER'")
     checkNereidsExecute("set experimental_enable_agg_state='true'")
-    checkNereidsExecute("set deprecated_enable_local_exchange = false")
+    checkNereidsExecute("set deprecated_group_by_and_having_use_alias_first = false")
     checkNereidsExecute("set show_hidden_columns=true")
     checkNereidsExecute("UNSET VARIABLE ALL")
     qt_cmd """show session variables like 'runtime_filter_type'"""
     qt_cmd """show session variables like 'experimental_enable_agg_state'"""
-    qt_cmd """show session variables like 'deprecated_enable_local_exchange'"""
+    qt_cmd """show session variables like 'deprecated_group_by_and_having_use_alias_first'"""
     qt_cmd """show session variables like 'show_hidden_columns'"""
 
     qt_cmd """select * from information_schema.session_variables where variable_name = 'show_hidden_columns'"""
@@ -82,12 +82,12 @@ suite("set_and_unset_variable_command", "nonConcurrent") {
     // test UNSET GLOBAL VARIABLE ALL
     checkNereidsExecute("set global runtime_filter_type='BLOOM_FILTER'")
     checkNereidsExecute("set global experimental_enable_agg_state='true'")
-    checkNereidsExecute("set global deprecated_enable_local_exchange = false")
+    checkNereidsExecute("set global deprecated_group_by_and_having_use_alias_first = false")
     checkNereidsExecute("set show_hidden_columns=true")
     checkNereidsExecute("UNSET global VARIABLE ALL")
     qt_cmd """show global variables like 'runtime_filter_type'"""
     qt_cmd """show global variables like 'experimental_enable_agg_state'"""
-    qt_cmd """show global variables like 'deprecated_enable_local_exchange'"""
+    qt_cmd """show global variables like 'deprecated_group_by_and_having_use_alias_first'"""
     qt_cmd """show global variables like 'show_hidden_columns'"""
 
     qt_cmd """select * from information_schema.global_variables where variable_name = 'show_hidden_columns'"""

From 7a0dd2370592f9190fecde4e9249a0db56ab8996 Mon Sep 17 00:00:00 2001
From: wangbo <wangbo@selectdb.com>
Date: Mon, 13 Jan 2025 10:07:31 +0800
Subject: [PATCH 128/140] [Refactor]Refactor workload group metric (#46640)

---
 .../workload_group/workload_group_metrics.cpp | 64 +++++++++----------
 .../workload_group/workload_group_metrics.h   | 23 ++++---
 2 files changed, 41 insertions(+), 46 deletions(-)

diff --git a/be/src/runtime/workload_group/workload_group_metrics.cpp b/be/src/runtime/workload_group/workload_group_metrics.cpp
index 77d7875b572428c..2a34427a490eda4 100644
--- a/be/src/runtime/workload_group/workload_group_metrics.cpp
+++ b/be/src/runtime/workload_group/workload_group_metrics.cpp
@@ -26,6 +26,13 @@
 
 namespace doris {
 
+DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(workload_group_cpu_time_sec, doris::MetricUnit::SECONDS);
+DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(workload_group_mem_used_bytes, doris::MetricUnit::BYTES);
+DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(workload_group_remote_scan_bytes, doris::MetricUnit::BYTES);
+DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(workload_group_total_local_scan_bytes,
+                                     doris::MetricUnit::BYTES);
+DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(workload_group_local_scan_bytes, doris::MetricUnit::BYTES);
+
 #include "common/compile_check_begin.h"
 
 WorkloadGroupMetrics::~WorkloadGroupMetrics() {
@@ -33,38 +40,27 @@ WorkloadGroupMetrics::~WorkloadGroupMetrics() {
 }
 
 WorkloadGroupMetrics::WorkloadGroupMetrics(WorkloadGroup* wg) {
+    std::string wg_id_prefix = "workload_group_" + std::to_string(wg->id());
     _entity = DorisMetrics::instance()->metric_registry()->register_entity(
-            "workload_group." + std::to_string(wg->id()), {{"workload_group", wg->name()}});
-
-    _cpu_time_metric = std::make_unique<doris::MetricPrototype>(
-            doris::MetricType::COUNTER, doris::MetricUnit::SECONDS, "workload_group_cpu_time_sec");
-    _cpu_time_counter = (IntCounter*)(_entity->register_metric<IntCounter>(_cpu_time_metric.get()));
-
-    _mem_used_bytes_metric = std::make_unique<doris::MetricPrototype>(
-            doris::MetricType::COUNTER, doris::MetricUnit::BYTES, "workload_group_mem_used_bytes");
-    _mem_used_bytes_counter =
-            (IntCounter*)(_entity->register_metric<IntCounter>(_mem_used_bytes_metric.get()));
-
-    _local_scan_bytes_metric = std::make_unique<doris::MetricPrototype>(
-            doris::MetricType::COUNTER, doris::MetricUnit::BYTES,
-            "workload_group_local_scan_bytes");
-    _local_scan_bytes_counter =
-            (IntCounter*)(_entity->register_metric<IntCounter>(_local_scan_bytes_metric.get()));
+            wg_id_prefix, {{"workload_group", wg->name()}, {"id", std::to_string(wg->id())}});
 
-    _remote_scan_bytes_metric = std::make_unique<doris::MetricPrototype>(
-            doris::MetricType::COUNTER, doris::MetricUnit::BYTES,
-            "workload_group_remote_scan_bytes");
-    _remote_scan_bytes_counter =
-            (IntCounter*)(_entity->register_metric<IntCounter>(_remote_scan_bytes_metric.get()));
+    INT_COUNTER_METRIC_REGISTER(_entity, workload_group_cpu_time_sec);
+    INT_GAUGE_METRIC_REGISTER(_entity, workload_group_mem_used_bytes);
+    INT_COUNTER_METRIC_REGISTER(_entity, workload_group_remote_scan_bytes);
+    INT_COUNTER_METRIC_REGISTER(_entity, workload_group_total_local_scan_bytes);
 
     std::vector<DataDirInfo>& data_dir_list = io::BeConfDataDirReader::be_config_data_dir_list;
     for (const auto& data_dir : data_dir_list) {
-        std::unique_ptr<doris::MetricPrototype> metric = std::make_unique<doris::MetricPrototype>(
-                doris::MetricType::COUNTER, doris::MetricUnit::BYTES,
-                "workload_group_local_scan_bytes_" + data_dir.metric_name);
-        _local_scan_bytes_counter_map.insert(
-                {data_dir.path, (IntCounter*)(_entity->register_metric<IntCounter>(metric.get()))});
-        _local_scan_bytes_metric_map.insert({data_dir.path, std::move(metric)});
+        std::string data_dir_metric_name = wg_id_prefix + "_io_" + data_dir.metric_name;
+        std::shared_ptr<MetricEntity> io_entity =
+                DorisMetrics::instance()->metric_registry()->register_entity(
+                        data_dir_metric_name, {{"workload_group", wg->name()},
+                                               {"path", data_dir.metric_name},
+                                               {"id", std::to_string(wg->id())}});
+        IntCounter* workload_group_local_scan_bytes = nullptr;
+        INT_COUNTER_METRIC_REGISTER(io_entity, workload_group_local_scan_bytes);
+        _local_scan_bytes_counter_map.insert({data_dir.path, workload_group_local_scan_bytes});
+        _io_entity_list.push_back(io_entity);
     }
 }
 
@@ -77,7 +73,7 @@ void WorkloadGroupMetrics::update_memory_used_bytes(int64_t memory_used) {
 }
 
 void WorkloadGroupMetrics::update_local_scan_io_bytes(std::string path, uint64_t delta_io_bytes) {
-    _local_scan_bytes_counter->increment(delta_io_bytes);
+    workload_group_total_local_scan_bytes->increment(delta_io_bytes);
     auto range = _local_scan_bytes_counter_map.equal_range(path);
     for (auto it = range.first; it != range.second; ++it) {
         it->second->increment((int64_t)delta_io_bytes);
@@ -85,7 +81,7 @@ void WorkloadGroupMetrics::update_local_scan_io_bytes(std::string path, uint64_t
 }
 
 void WorkloadGroupMetrics::update_remote_scan_io_bytes(uint64_t delta_io_bytes) {
-    _remote_scan_bytes_counter->increment(delta_io_bytes);
+    workload_group_remote_scan_bytes->increment(delta_io_bytes);
 }
 
 void WorkloadGroupMetrics::refresh_metrics() {
@@ -94,21 +90,21 @@ void WorkloadGroupMetrics::refresh_metrics() {
     // cpu
     uint64_t _current_cpu_time_nanos = _cpu_time_nanos.load();
     uint64_t _cpu_time_sec = _current_cpu_time_nanos / (1000L * 1000L * 1000L);
-    _cpu_time_counter->set_value(_cpu_time_sec);
+    workload_group_cpu_time_sec->set_value(_cpu_time_sec);
     _per_sec_cpu_time_nanos = (_current_cpu_time_nanos - _last_cpu_time_nanos) / interval_second;
     _last_cpu_time_nanos = _current_cpu_time_nanos;
 
     // memory
-    _mem_used_bytes_counter->set_value(_memory_used);
+    workload_group_mem_used_bytes->set_value(_memory_used);
 
     // local scan
-    int64_t current_local_scan_bytes = _local_scan_bytes_counter->value();
+    int64_t current_local_scan_bytes = workload_group_total_local_scan_bytes->value();
     _per_sec_local_scan_bytes =
             (current_local_scan_bytes - _last_local_scan_bytes) / interval_second;
     _last_local_scan_bytes = current_local_scan_bytes;
 
     // remote scan
-    int64_t current_remote_scan_bytes = _remote_scan_bytes_counter->value();
+    int64_t current_remote_scan_bytes = workload_group_remote_scan_bytes->value();
     _per_sec_remote_scan_bytes =
             (current_remote_scan_bytes - _last_remote_scan_bytes) / interval_second;
     _last_remote_scan_bytes = current_remote_scan_bytes;
@@ -127,7 +123,7 @@ int64_t WorkloadGroupMetrics::get_remote_scan_bytes_per_second() {
 }
 
 int64_t WorkloadGroupMetrics::get_memory_used() {
-    return _mem_used_bytes_counter->value();
+    return workload_group_mem_used_bytes->value();
 }
 
 } // namespace doris
\ No newline at end of file
diff --git a/be/src/runtime/workload_group/workload_group_metrics.h b/be/src/runtime/workload_group/workload_group_metrics.h
index 7ac6ddbb1f3a700..67085d8374fa1d1 100644
--- a/be/src/runtime/workload_group/workload_group_metrics.h
+++ b/be/src/runtime/workload_group/workload_group_metrics.h
@@ -22,6 +22,7 @@
 #include <memory>
 #include <string>
 #include <unordered_map>
+#include <vector>
 
 namespace doris {
 
@@ -30,6 +31,11 @@ class WorkloadGroup;
 template <typename T>
 class AtomicCounter;
 using IntCounter = AtomicCounter<int64_t>;
+
+template <typename T>
+class AtomicGauge;
+using IntGuage = AtomicGauge<int64_t>;
+
 class MetricEntity;
 struct MetricPrototype;
 
@@ -58,18 +64,10 @@ class WorkloadGroupMetrics {
     int64_t get_memory_used();
 
 private:
-    std::unique_ptr<doris::MetricPrototype> _cpu_time_metric {nullptr};
-    std::unique_ptr<doris::MetricPrototype> _mem_used_bytes_metric {nullptr};
-    std::unique_ptr<doris::MetricPrototype> _local_scan_bytes_metric {nullptr};
-    std::unique_ptr<doris::MetricPrototype> _remote_scan_bytes_metric {nullptr};
-    // NOTE: _local_scan_bytes_metric is sum of all disk's IO
-    std::unordered_multimap<std::string, std::unique_ptr<doris::MetricPrototype>>
-            _local_scan_bytes_metric_map;
-
-    IntCounter* _cpu_time_counter {nullptr};          // used for metric
-    IntCounter* _mem_used_bytes_counter {nullptr};    // used for metric
-    IntCounter* _local_scan_bytes_counter {nullptr};  // used for metric
-    IntCounter* _remote_scan_bytes_counter {nullptr}; // used for metric
+    IntCounter* workload_group_cpu_time_sec {nullptr};           // used for metric
+    IntGuage* workload_group_mem_used_bytes {nullptr};           // used for metric
+    IntCounter* workload_group_remote_scan_bytes {nullptr};      // used for metric
+    IntCounter* workload_group_total_local_scan_bytes {nullptr}; // used for metric
     std::unordered_multimap<std::string, IntCounter*>
             _local_scan_bytes_counter_map; // used for metric
 
@@ -86,6 +84,7 @@ class WorkloadGroupMetrics {
     std::atomic<uint64_t> _memory_used {0};
 
     std::shared_ptr<MetricEntity> _entity {nullptr};
+    std::vector<std::shared_ptr<MetricEntity>> _io_entity_list;
 };
 
 } // namespace doris
\ No newline at end of file

From ff211c3669cba9b9fa943638cf6ef81e220ed5cf Mon Sep 17 00:00:00 2001
From: Gabriel <liwenqiang@selectdb.com>
Date: Mon, 13 Jan 2025 10:13:09 +0800
Subject: [PATCH 129/140] [test](UT) Add UT to test pipeline initialization and
 execution (#46718)

---
 .../exec/exchange_source_operator.cpp         |   6 +-
 be/src/pipeline/pipeline_fragment_context.cpp |   9 +-
 be/src/pipeline/pipeline_fragment_context.h   |   2 +-
 be/src/pipeline/pipeline_task.cpp             |  16 +-
 be/src/pipeline/pipeline_task.h               |   4 +-
 be/src/runtime/exec_env.h                     |   1 +
 be/src/runtime/exec_env_init.cpp              |   2 +-
 be/src/runtime/query_context.cpp              |  15 +-
 be/src/vec/runtime/vdata_stream_mgr.cpp       |  12 +-
 be/src/vec/runtime/vdata_stream_mgr.h         |  12 +-
 be/src/vec/runtime/vdata_stream_recvr.cpp     |  11 +-
 be/src/vec/runtime/vdata_stream_recvr.h       |   4 +-
 be/test/pipeline/pipeline_test.cpp            | 340 +++++++++++++++++
 be/test/pipeline/thrift_builder.h             | 354 ++++++++++++++++++
 14 files changed, 746 insertions(+), 42 deletions(-)
 create mode 100644 be/test/pipeline/pipeline_test.cpp
 create mode 100644 be/test/pipeline/thrift_builder.h

diff --git a/be/src/pipeline/exec/exchange_source_operator.cpp b/be/src/pipeline/exec/exchange_source_operator.cpp
index dbde9abd05dc347..6ac0b9ee630e924 100644
--- a/be/src/pipeline/exec/exchange_source_operator.cpp
+++ b/be/src/pipeline/exec/exchange_source_operator.cpp
@@ -63,8 +63,8 @@ Status ExchangeLocalState::init(RuntimeState* state, LocalStateInfo& info) {
     SCOPED_TIMER(_init_timer);
     auto& p = _parent->cast<ExchangeSourceOperatorX>();
     stream_recvr = state->exec_env()->vstream_mgr()->create_recvr(
-            state, this, p.input_row_desc(), state->fragment_instance_id(), p.node_id(),
-            p.num_senders(), profile(), p.is_merging());
+            state, _memory_used_counter, p.input_row_desc(), state->fragment_instance_id(),
+            p.node_id(), p.num_senders(), profile(), p.is_merging());
     const auto& queues = stream_recvr->sender_queues();
     deps.resize(queues.size());
     metrics.resize(queues.size());
@@ -132,8 +132,6 @@ Status ExchangeSourceOperatorX::open(RuntimeState* state) {
 
     if (_is_merging) {
         RETURN_IF_ERROR(_vsort_exec_exprs.prepare(state, _row_descriptor, _row_descriptor));
-    }
-    if (_is_merging) {
         RETURN_IF_ERROR(_vsort_exec_exprs.open(state));
     }
     return Status::OK();
diff --git a/be/src/pipeline/pipeline_fragment_context.cpp b/be/src/pipeline/pipeline_fragment_context.cpp
index e2ad5e0d14cc428..ddcdce460feb960 100644
--- a/be/src/pipeline/pipeline_fragment_context.cpp
+++ b/be/src/pipeline/pipeline_fragment_context.cpp
@@ -499,8 +499,13 @@ Status PipelineFragmentContext::_build_pipeline_tasks(const doris::TPipelineFrag
             if (pipeline_id_to_task.contains(_pipelines[pip_idx]->id())) {
                 auto* task = pipeline_id_to_task[_pipelines[pip_idx]->id()];
                 DCHECK(pipeline_id_to_profile[pip_idx]);
-                RETURN_IF_ERROR_OR_CATCH_EXCEPTION(task->prepare(
-                        local_params, request.fragment.output_sink, _query_ctx.get()));
+                std::vector<TScanRangeParams> scan_ranges;
+                scan_ranges = find_with_default(local_params.per_node_scan_ranges,
+                                                _pipelines[pip_idx]->operators().front()->node_id(),
+                                                scan_ranges);
+                RETURN_IF_ERROR_OR_CATCH_EXCEPTION(
+                        task->prepare(scan_ranges, local_params.sender_id,
+                                      request.fragment.output_sink, _query_ctx.get()));
             }
         }
         {
diff --git a/be/src/pipeline/pipeline_fragment_context.h b/be/src/pipeline/pipeline_fragment_context.h
index 1674afa886d520d..d672ad6e9233c12 100644
--- a/be/src/pipeline/pipeline_fragment_context.h
+++ b/be/src/pipeline/pipeline_fragment_context.h
@@ -84,7 +84,7 @@ class PipelineFragmentContext : public TaskExecutionContext {
 
     QueryContext* get_query_ctx() { return _query_ctx.get(); }
     // should be protected by lock?
-    [[nodiscard]] bool is_canceled() const { return _runtime_state->is_cancelled(); }
+    [[nodiscard]] bool is_canceled() const { return _query_ctx->is_cancelled(); }
 
     Status prepare(const doris::TPipelineFragmentParams& request, ThreadPool* thread_pool);
 
diff --git a/be/src/pipeline/pipeline_task.cpp b/be/src/pipeline/pipeline_task.cpp
index 5ed725010ec3647..a8e3ecbd82abf3f 100644
--- a/be/src/pipeline/pipeline_task.cpp
+++ b/be/src/pipeline/pipeline_task.cpp
@@ -74,8 +74,8 @@ PipelineTask::PipelineTask(
     }
 }
 
-Status PipelineTask::prepare(const TPipelineInstanceParams& local_params, const TDataSink& tsink,
-                             QueryContext* query_ctx) {
+Status PipelineTask::prepare(const std::vector<TScanRangeParams>& scan_range, const int sender_id,
+                             const TDataSink& tsink, QueryContext* query_ctx) {
     DCHECK(_sink);
     _init_profile();
     SCOPED_TIMER(_task_profile->total_time_counter());
@@ -87,17 +87,13 @@ Status PipelineTask::prepare(const TPipelineInstanceParams& local_params, const
     });
     {
         // set sink local state
-        LocalSinkStateInfo info {_task_idx,
-                                 _task_profile.get(),
-                                 local_params.sender_id,
-                                 get_sink_shared_state().get(),
-                                 _le_state_map,
-                                 tsink};
+        LocalSinkStateInfo info {_task_idx,     _task_profile.get(),
+                                 sender_id,     get_sink_shared_state().get(),
+                                 _le_state_map, tsink};
         RETURN_IF_ERROR(_sink->setup_local_state(_state, info));
     }
 
-    _scan_ranges = find_with_default(local_params.per_node_scan_ranges,
-                                     _operators.front()->node_id(), _scan_ranges);
+    _scan_ranges = scan_range;
     auto* parent_profile = _state->get_sink_local_state()->profile();
     query_ctx->register_query_statistics(
             _state->get_sink_local_state()->get_query_statistics_ptr());
diff --git a/be/src/pipeline/pipeline_task.h b/be/src/pipeline/pipeline_task.h
index 1a31e5954f479c4..6143feb5a81bf99 100644
--- a/be/src/pipeline/pipeline_task.h
+++ b/be/src/pipeline/pipeline_task.h
@@ -54,8 +54,8 @@ class PipelineTask {
                          le_state_map,
                  int task_idx);
 
-    Status prepare(const TPipelineInstanceParams& local_params, const TDataSink& tsink,
-                   QueryContext* query_ctx);
+    Status prepare(const std::vector<TScanRangeParams>& scan_range, const int sender_id,
+                   const TDataSink& tsink, QueryContext* query_ctx);
 
     Status execute(bool* eos);
 
diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h
index 12d625da3bfcae1..e7bdc1b785d2806 100644
--- a/be/src/runtime/exec_env.h
+++ b/be/src/runtime/exec_env.h
@@ -347,6 +347,7 @@ class ExecEnv {
     arrow::MemoryPool* arrow_memory_pool() { return _arrow_memory_pool; }
 
     bool check_auth_token(const std::string& auth_token);
+    void set_stream_mgr(vectorized::VDataStreamMgr* vstream_mgr) { _vstream_mgr = vstream_mgr; }
 
 private:
     ExecEnv();
diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index f0d8253254fe41b..b7f2e840dd94b1d 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -192,7 +192,7 @@ Status ExecEnv::_init(const std::vector<StorePath>& store_paths,
     _user_function_cache = new UserFunctionCache();
     static_cast<void>(_user_function_cache->init(doris::config::user_function_dir));
     _external_scan_context_mgr = new ExternalScanContextMgr(this);
-    _vstream_mgr = new doris::vectorized::VDataStreamMgr();
+    set_stream_mgr(new doris::vectorized::VDataStreamMgr());
     _result_mgr = new ResultBufferMgr();
     _result_queue_mgr = new ResultQueueMgr();
     _backend_client_cache = new BackendServiceClientCache(config::max_client_cache_size_per_host);
diff --git a/be/src/runtime/query_context.cpp b/be/src/runtime/query_context.cpp
index 1ff08f7d9bb8765..0e95b37d5169818 100644
--- a/be/src/runtime/query_context.cpp
+++ b/be/src/runtime/query_context.cpp
@@ -153,12 +153,14 @@ QueryContext::~QueryContext() {
                 MemCounter::print_bytes(query_mem_tracker->consumption()),
                 MemCounter::print_bytes(query_mem_tracker->peak_consumption()));
     }
-    uint64_t group_id = 0;
+    [[maybe_unused]] uint64_t group_id = 0;
     if (_workload_group) {
         group_id = _workload_group->id(); // before remove
     }
 
+#ifndef BE_TEST
     _exec_env->runtime_query_statistics_mgr()->set_query_finished(print_id(_query_id));
+#endif
 
     if (enable_profile()) {
         _report_query_profile();
@@ -176,7 +178,7 @@ QueryContext::~QueryContext() {
                          << print_id(_query_id) << ", error status " << submit_st;
         }
     }
-
+#ifndef BE_TEST
     if (ExecEnv::GetInstance()->pipeline_tracer_context()->enabled()) [[unlikely]] {
         try {
             ExecEnv::GetInstance()->pipeline_tracer_context()->end_query(_query_id, group_id);
@@ -184,6 +186,7 @@ QueryContext::~QueryContext() {
             LOG(WARNING) << "Dump trace log failed bacause " << e.what();
         }
     }
+#endif
     _runtime_filter_mgr.reset();
     _execution_dependency.reset();
     _shared_hash_table_controller.reset();
@@ -192,7 +195,9 @@ QueryContext::~QueryContext() {
     obj_pool.clear();
     _merge_controller_handler.reset();
 
+#ifndef BE_TEST
     _exec_env->spill_stream_mgr()->async_cleanup_query(_query_id);
+#endif
     DorisMetrics::instance()->query_ctx_cnt->increment(-1);
     // the only one msg shows query's end. any other msg should append to it if need.
     LOG_INFO("Query {} deconstructed, mem_tracker: {}", print_id(this->_query_id), mem_tracker_msg);
@@ -275,8 +280,10 @@ void QueryContext::set_pipeline_context(
 }
 
 void QueryContext::register_query_statistics(std::shared_ptr<QueryStatistics> qs) {
+#ifndef BE_TEST
     _exec_env->runtime_query_statistics_mgr()->register_query_statistics(
             print_id(_query_id), qs, current_connect_fe, _query_options.query_type);
+#endif
 }
 
 std::shared_ptr<QueryStatistics> QueryContext::get_query_statistics() {
@@ -289,8 +296,10 @@ void QueryContext::register_memory_statistics() {
         std::shared_ptr<QueryStatistics> qs = query_mem_tracker->get_query_statistics();
         std::string query_id = print_id(_query_id);
         if (qs) {
+#ifndef BE_TEST
             _exec_env->runtime_query_statistics_mgr()->register_query_statistics(
                     query_id, qs, current_connect_fe, _query_options.query_type);
+#endif
         } else {
             LOG(INFO) << " query " << query_id << " get memory query statistics failed ";
         }
@@ -300,9 +309,11 @@ void QueryContext::register_memory_statistics() {
 void QueryContext::register_cpu_statistics() {
     if (!_cpu_statistics) {
         _cpu_statistics = std::make_shared<QueryStatistics>();
+#ifndef BE_TEST
         _exec_env->runtime_query_statistics_mgr()->register_query_statistics(
                 print_id(_query_id), _cpu_statistics, current_connect_fe,
                 _query_options.query_type);
+#endif
     }
 }
 
diff --git a/be/src/vec/runtime/vdata_stream_mgr.cpp b/be/src/vec/runtime/vdata_stream_mgr.cpp
index 7dad3d2c86705e7..43420ad7ad66fce 100644
--- a/be/src/vec/runtime/vdata_stream_mgr.cpp
+++ b/be/src/vec/runtime/vdata_stream_mgr.cpp
@@ -64,15 +64,15 @@ inline uint32_t VDataStreamMgr::get_hash_value(const TUniqueId& fragment_instanc
 }
 
 std::shared_ptr<VDataStreamRecvr> VDataStreamMgr::create_recvr(
-        RuntimeState* state, pipeline::ExchangeLocalState* parent, const RowDescriptor& row_desc,
-        const TUniqueId& fragment_instance_id, PlanNodeId dest_node_id, int num_senders,
-        RuntimeProfile* profile, bool is_merging) {
+        RuntimeState* state, RuntimeProfile::HighWaterMarkCounter* memory_used_counter,
+        const RowDescriptor& row_desc, const TUniqueId& fragment_instance_id,
+        PlanNodeId dest_node_id, int num_senders, RuntimeProfile* profile, bool is_merging) {
     DCHECK(profile != nullptr);
     VLOG_FILE << "creating receiver for fragment=" << print_id(fragment_instance_id)
               << ", node=" << dest_node_id;
-    std::shared_ptr<VDataStreamRecvr> recvr(new VDataStreamRecvr(this, parent, state, row_desc,
-                                                                 fragment_instance_id, dest_node_id,
-                                                                 num_senders, is_merging, profile));
+    std::shared_ptr<VDataStreamRecvr> recvr(
+            new VDataStreamRecvr(this, memory_used_counter, state, row_desc, fragment_instance_id,
+                                 dest_node_id, num_senders, is_merging, profile));
     uint32_t hash_value = get_hash_value(fragment_instance_id, dest_node_id);
     std::unique_lock l(_lock);
     _fragment_stream_set.insert(std::make_pair(fragment_instance_id, dest_node_id));
diff --git a/be/src/vec/runtime/vdata_stream_mgr.h b/be/src/vec/runtime/vdata_stream_mgr.h
index 81f9d19c6b6aebb..61d831633bd9b23 100644
--- a/be/src/vec/runtime/vdata_stream_mgr.h
+++ b/be/src/vec/runtime/vdata_stream_mgr.h
@@ -29,6 +29,7 @@
 
 #include "common/global_types.h"
 #include "common/status.h"
+#include "util/runtime_profile.h"
 
 namespace google {
 #include "common/compile_check_begin.h"
@@ -40,7 +41,6 @@ class Closure;
 namespace doris {
 class RuntimeState;
 class RowDescriptor;
-class RuntimeProfile;
 class PTransmitDataParams;
 namespace pipeline {
 class ExchangeLocalState;
@@ -54,12 +54,10 @@ class VDataStreamMgr {
     VDataStreamMgr();
     ~VDataStreamMgr();
 
-    std::shared_ptr<VDataStreamRecvr> create_recvr(RuntimeState* state,
-                                                   pipeline::ExchangeLocalState* parent,
-                                                   const RowDescriptor& row_desc,
-                                                   const TUniqueId& fragment_instance_id,
-                                                   PlanNodeId dest_node_id, int num_senders,
-                                                   RuntimeProfile* profile, bool is_merging);
+    std::shared_ptr<VDataStreamRecvr> create_recvr(
+            RuntimeState* state, RuntimeProfile::HighWaterMarkCounter* memory_used_counter,
+            const RowDescriptor& row_desc, const TUniqueId& fragment_instance_id,
+            PlanNodeId dest_node_id, int num_senders, RuntimeProfile* profile, bool is_merging);
 
     Status find_recvr(const TUniqueId& fragment_instance_id, PlanNodeId node_id,
                       std::shared_ptr<VDataStreamRecvr>* res, bool acquire_lock = true);
diff --git a/be/src/vec/runtime/vdata_stream_recvr.cpp b/be/src/vec/runtime/vdata_stream_recvr.cpp
index c3277b0917e84c6..e2cadb37eec841b 100644
--- a/be/src/vec/runtime/vdata_stream_recvr.cpp
+++ b/be/src/vec/runtime/vdata_stream_recvr.cpp
@@ -103,7 +103,7 @@ Status VDataStreamRecvr::SenderQueue::get_batch(Block* block, bool* eos) {
     COUNTER_UPDATE(_recvr->_deserialize_row_batch_timer, block_item.deserialize_time());
     COUNTER_UPDATE(_recvr->_decompress_timer, block->get_decompress_time());
     COUNTER_UPDATE(_recvr->_decompress_bytes, block->get_decompressed_bytes());
-    _recvr->_parent->memory_used_counter()->update(-(int64_t)block_byte_size);
+    _recvr->_memory_used_counter->update(-(int64_t)block_byte_size);
     std::lock_guard<std::mutex> l(_lock);
     sub_blocks_memory_usage(block_byte_size);
     _record_debug_info();
@@ -198,7 +198,7 @@ Status VDataStreamRecvr::SenderQueue::add_block(std::unique_ptr<PBlock> pblock,
         _pending_closures.emplace_back(*done, monotonicStopWatch);
         *done = nullptr;
     }
-    _recvr->_parent->memory_used_counter()->update(block_byte_size);
+    _recvr->_memory_used_counter->update(block_byte_size);
     add_blocks_memory_usage(block_byte_size);
     return Status::OK();
 }
@@ -237,7 +237,7 @@ void VDataStreamRecvr::SenderQueue::add_block(Block* block, bool use_move) {
         _record_debug_info();
         try_set_dep_ready_without_lock();
         COUNTER_UPDATE(_recvr->_local_bytes_received_counter, block_mem_size);
-        _recvr->_parent->memory_used_counter()->update(block_mem_size);
+        _recvr->_memory_used_counter->update(block_mem_size);
         add_blocks_memory_usage(block_mem_size);
     }
 }
@@ -308,13 +308,14 @@ void VDataStreamRecvr::SenderQueue::close() {
     _block_queue.clear();
 }
 
-VDataStreamRecvr::VDataStreamRecvr(VDataStreamMgr* stream_mgr, pipeline::ExchangeLocalState* parent,
+VDataStreamRecvr::VDataStreamRecvr(VDataStreamMgr* stream_mgr,
+                                   RuntimeProfile::HighWaterMarkCounter* memory_used_counter,
                                    RuntimeState* state, const RowDescriptor& row_desc,
                                    const TUniqueId& fragment_instance_id, PlanNodeId dest_node_id,
                                    int num_senders, bool is_merging, RuntimeProfile* profile)
         : HasTaskExecutionCtx(state),
           _mgr(stream_mgr),
-          _parent(parent),
+          _memory_used_counter(memory_used_counter),
           _query_thread_context(state->query_id(), state->query_mem_tracker(),
                                 state->get_query_ctx()->workload_group()),
           _fragment_instance_id(fragment_instance_id),
diff --git a/be/src/vec/runtime/vdata_stream_recvr.h b/be/src/vec/runtime/vdata_stream_recvr.h
index 1639366c8b83d68..d35de6bd4596e78 100644
--- a/be/src/vec/runtime/vdata_stream_recvr.h
+++ b/be/src/vec/runtime/vdata_stream_recvr.h
@@ -71,7 +71,7 @@ class VDataStreamRecvr;
 class VDataStreamRecvr : public HasTaskExecutionCtx {
 public:
     class SenderQueue;
-    VDataStreamRecvr(VDataStreamMgr* stream_mgr, pipeline::ExchangeLocalState* parent,
+    VDataStreamRecvr(VDataStreamMgr* stream_mgr, RuntimeProfile::HighWaterMarkCounter* counter,
                      RuntimeState* state, const RowDescriptor& row_desc,
                      const TUniqueId& fragment_instance_id, PlanNodeId dest_node_id,
                      int num_senders, bool is_merging, RuntimeProfile* profile);
@@ -123,7 +123,7 @@ class VDataStreamRecvr : public HasTaskExecutionCtx {
     // DataStreamMgr instance used to create this recvr. (Not owned)
     VDataStreamMgr* _mgr = nullptr;
 
-    pipeline::ExchangeLocalState* _parent = nullptr;
+    RuntimeProfile::HighWaterMarkCounter* _memory_used_counter = nullptr;
 
     QueryThreadContext _query_thread_context;
 
diff --git a/be/test/pipeline/pipeline_test.cpp b/be/test/pipeline/pipeline_test.cpp
new file mode 100644
index 000000000000000..600183fe3ac6309
--- /dev/null
+++ b/be/test/pipeline/pipeline_test.cpp
@@ -0,0 +1,340 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "pipeline/pipeline.h"
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include "common/exception.h"
+#include "common/status.h"
+#include "pipeline/dependency.h"
+#include "pipeline/exec/exchange_source_operator.h"
+#include "pipeline/pipeline_fragment_context.h"
+#include "pipeline/task_queue.h"
+#include "runtime/exec_env.h"
+#include "runtime/fragment_mgr.h"
+#include "thrift_builder.h"
+#include "vec/columns/column.h"
+#include "vec/columns/column_vector.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/runtime/vdata_stream_mgr.h"
+
+namespace doris::pipeline {
+
+static void empty_function(RuntimeState*, Status*) {}
+
+class PipelineTest : public testing::Test {
+public:
+    PipelineTest() : _obj_pool(new ObjectPool()) {
+        _query_options = TQueryOptionsBuilder().set_enable_local_exchange(true).build();
+        auto fe_address = TNetworkAddress();
+        fe_address.hostname = LOCALHOST;
+        fe_address.port = DUMMY_PORT;
+        _query_ctx = QueryContext::create_shared(_query_id, ExecEnv::GetInstance(), _query_options,
+                                                 fe_address, true, fe_address,
+                                                 QuerySource::INTERNAL_FRONTEND);
+        ExecEnv::GetInstance()->set_stream_mgr(new doris::vectorized::VDataStreamMgr());
+        _task_queue = std::make_unique<MultiCoreTaskQueue>(1);
+    }
+    ~PipelineTest() override = default;
+
+private:
+    std::shared_ptr<Pipeline> _build_pipeline(int id, int num_instances,
+                                              Pipeline* parent = nullptr) {
+        auto pip = std::make_shared<Pipeline>(
+                id, parent ? std::min(parent->num_tasks(), num_instances) : num_instances,
+                parent ? parent->num_tasks() : num_instances);
+        _pipelines.push_back(pip);
+        _pipeline_tasks.push_back(std::vector<std::unique_ptr<PipelineTask>> {});
+        _runtime_states.push_back(std::vector<std::unique_ptr<RuntimeState>> {});
+        _pipeline_profiles.push_back(nullptr);
+        return pip;
+    }
+    std::shared_ptr<PipelineFragmentContext> _build_fragment_context() {
+        int fragment_id = _next_fragment_id();
+        _context.push_back(std::make_shared<PipelineFragmentContext>(
+                _query_id, fragment_id, _query_ctx, ExecEnv::GetInstance(), empty_function,
+                std::bind<Status>(std::mem_fn(&FragmentMgr::trigger_pipeline_context_report),
+                                  ExecEnv::GetInstance()->fragment_mgr(), std::placeholders::_1,
+                                  std::placeholders::_2)));
+        _runtime_state.push_back(RuntimeState::create_unique(
+                _query_id, fragment_id, _query_options, _query_ctx->query_globals,
+                ExecEnv::GetInstance(), _query_ctx.get()));
+        _runtime_state.back()->set_task_execution_context(
+                std::static_pointer_cast<TaskExecutionContext>(_context.back()));
+        return _context.back();
+    }
+    TUniqueId _next_ins_id() {
+        _ins_id.lo++;
+        return _ins_id;
+    }
+    int _next_fragment_id() { return _fragment_id++; }
+    int _next_node_id() { return _next_node_idx++; }
+    int _next_op_id() { return _next_op_idx++; }
+
+    // Query level
+    std::shared_ptr<ObjectPool> _obj_pool;
+    std::shared_ptr<QueryContext> _query_ctx;
+    TUniqueId _query_id = TUniqueId();
+    TQueryOptions _query_options;
+    std::unique_ptr<MultiCoreTaskQueue> _task_queue;
+
+    // Fragment level
+    // Fragment0 -> Fragment1
+    std::vector<std::unique_ptr<RuntimeState>> _runtime_state;
+    std::vector<std::shared_ptr<PipelineFragmentContext>> _context;
+    int _fragment_id = 0;
+
+    TUniqueId _ins_id = TUniqueId();
+    int _next_node_idx = 0;
+    int _next_op_idx = 0;
+
+    // Pipeline Level
+    // Fragment0[Pipeline0 -> Pipeline1] -> Fragment1[Pipeline2 -> Pipeline3]
+    std::vector<std::shared_ptr<Pipeline>> _pipelines;
+    std::vector<std::shared_ptr<RuntimeProfile>> _pipeline_profiles;
+
+    // Task Level
+    // Fragment0[Pipeline0[Task0] -> Pipeline1[Task0]] -> Fragment1[Pipeline2[Task0] -> Pipeline3[Task0]]
+    std::vector<std::vector<std::unique_ptr<PipelineTask>>> _pipeline_tasks;
+    std::vector<std::vector<std::unique_ptr<RuntimeState>>> _runtime_states;
+
+    const std::string LOCALHOST = BackendOptions::get_localhost();
+    const int DUMMY_PORT = config::brpc_port;
+};
+
+TEST_F(PipelineTest, HAPPY_PATH) {
+    // Pipeline(ExchangeOperator(id=0, UNPARTITIONED) -> ExchangeSinkOperatorX(id=1, UNPARTITIONED))
+    int parallelism = 1;
+
+    // Build pipeline
+    int pipeline_id = 0;
+    DescriptorTbl* desc;
+    OperatorPtr op;
+    _build_fragment_context();
+    auto cur_pipe = _build_pipeline(pipeline_id, parallelism);
+    {
+        auto tnode = TPlanNodeBuilder(_next_node_id(), TPlanNodeType::EXCHANGE_NODE)
+                             .set_exchange_node(
+                                     TExchangeNodeBuilder()
+                                             .set_partition_type(TPartitionType::UNPARTITIONED)
+                                             .append_input_row_tuples(0)
+                                             .build())
+                             .append_row_tuples(0, false)
+                             .build();
+
+        TTupleDescriptor tuple0 = TTupleDescriptorBuilder().set_id(0).build();
+        TSlotDescriptor slot0 =
+                TSlotDescriptorBuilder()
+                        .set_id(0)
+                        .set_parent(tuple0)
+                        .set_slotType(
+                                TTypeDescBuilder()
+                                        .set_types(TTypeNodeBuilder()
+                                                           .set_type(TTypeNodeType::SCALAR)
+                                                           .set_scalar_type(TPrimitiveType::INT)
+                                                           .build())
+                                        .build())
+                        .set_nullIndicatorBit(0)
+                        .set_byteOffset(0)
+                        .set_slotIdx(0)
+                        .set_isMaterialized(true)
+                        .set_colName("test_column0")
+                        .build();
+
+        TDescriptorTable desc_table = TDescriptorTableBuilder()
+                                              .append_slotDescriptors(slot0)
+                                              .append_tupleDescriptors(tuple0)
+                                              .build();
+
+        EXPECT_EQ(DescriptorTbl::create(_obj_pool.get(), desc_table, &desc), Status::OK());
+        op.reset(new ExchangeSourceOperatorX(_obj_pool.get(), tnode, _next_op_id(), *desc, 1));
+        EXPECT_EQ(op->init(tnode, _runtime_state.back().get()), Status::OK());
+        auto& exchange_operator = op->cast<ExchangeSourceOperatorX>();
+        EXPECT_EQ(exchange_operator._is_merging, false);
+        EXPECT_EQ(cur_pipe->add_operator(op, 0), Status::OK());
+    }
+
+    TDataSink tsink;
+    // 0-1
+    auto source_ins = _next_ins_id();
+    // 0-2
+    auto dest0 = _next_ins_id();
+    {
+        std::vector<TPlanFragmentDestination> destinations;
+        auto dest0_address = TNetworkAddress();
+        dest0_address.hostname = LOCALHOST;
+        dest0_address.port = DUMMY_PORT;
+        destinations.push_back(
+                TPlanFragmentDestinationBuilder(dest0, dest0_address, dest0_address).build());
+        auto stream_sink =
+                TDataStreamSinkBuilder(_next_node_id(),
+                                       TDataPartitionBuilder(TPartitionType::UNPARTITIONED).build())
+                        .build();
+        tsink = TDataSinkBuilder(TDataSinkType::DATA_STREAM_SINK)
+                        .set_stream_sink(stream_sink)
+                        .build();
+        DataSinkOperatorPtr sink;
+
+        std::vector<TUniqueId> ids;
+        ids.push_back(source_ins);
+        sink.reset(new ExchangeSinkOperatorX(_runtime_state.back().get(), op->row_desc(),
+                                             _next_op_id(), stream_sink, destinations, ids));
+        EXPECT_EQ(sink->init(tsink), Status::OK());
+        EXPECT_EQ(cur_pipe->set_sink(sink), Status::OK());
+
+        EXPECT_EQ(cur_pipe->sink()->set_child(cur_pipe->operators().back()), Status::OK());
+
+        EXPECT_EQ(cur_pipe->prepare(_runtime_state.back().get()), Status::OK());
+        EXPECT_EQ(cur_pipe->num_tasks(), parallelism);
+    }
+
+    {
+        // Build pipeline task
+        int task_id = 0;
+        std::unique_ptr<RuntimeState> local_runtime_state = RuntimeState::create_unique(
+                source_ins, _query_id, _context.back()->get_fragment_id(), _query_options,
+                _query_ctx->query_globals, ExecEnv::GetInstance(), _query_ctx.get());
+        local_runtime_state->set_desc_tbl(desc);
+        local_runtime_state->set_per_fragment_instance_idx(0);
+        local_runtime_state->set_num_per_fragment_instances(parallelism);
+        local_runtime_state->resize_op_id_to_local_state(-1);
+        local_runtime_state->set_max_operator_id(-1);
+        local_runtime_state->set_load_stream_per_node(0);
+        local_runtime_state->set_total_load_streams(0);
+        local_runtime_state->set_num_local_sink(0);
+        local_runtime_state->set_task_id(task_id);
+        local_runtime_state->set_task_num(cur_pipe->num_tasks());
+        local_runtime_state->set_task_execution_context(
+                std::static_pointer_cast<TaskExecutionContext>(_context.back()));
+
+        _pipeline_profiles[pipeline_id] =
+                std::make_shared<RuntimeProfile>("Pipeline : " + std::to_string(pipeline_id));
+        std::map<int,
+                 std::pair<std::shared_ptr<LocalExchangeSharedState>, std::shared_ptr<Dependency>>>
+                le_state_map;
+        auto task = std::make_unique<PipelineTask>(
+                cur_pipe, task_id, local_runtime_state.get(), _context.back().get(),
+                _pipeline_profiles[pipeline_id].get(), le_state_map, task_id);
+        cur_pipe->incr_created_tasks(task_id, task.get());
+        local_runtime_state->set_task(task.get());
+        task->set_task_queue(_task_queue.get());
+        _pipeline_tasks[pipeline_id].push_back(std::move(task));
+        _runtime_states[pipeline_id].push_back(std::move(local_runtime_state));
+    }
+
+    auto context = _build_fragment_context();
+    std::unique_ptr<RuntimeState> downstream_runtime_state = RuntimeState::create_unique(
+            dest0, _query_id, _next_fragment_id(), _query_options, _query_ctx->query_globals,
+            ExecEnv::GetInstance(), _query_ctx.get());
+    downstream_runtime_state->set_task_execution_context(
+            std::static_pointer_cast<TaskExecutionContext>(context));
+
+    auto downstream_pipeline_profile =
+            std::make_shared<RuntimeProfile>("Pipeline : " + std::to_string(pipeline_id));
+    auto* memory_used_counter = downstream_pipeline_profile->AddHighWaterMarkCounter(
+            "MemoryUsage", TUnit::BYTES, "", 1);
+    auto downstream_recvr = ExecEnv::GetInstance()->_vstream_mgr->create_recvr(
+            downstream_runtime_state.get(), memory_used_counter, op->row_desc(), dest0, 1, 1,
+            downstream_pipeline_profile.get(), false);
+    std::vector<TScanRangeParams> scan_ranges;
+    EXPECT_EQ(_pipeline_tasks[pipeline_id].back()->prepare(scan_ranges, 0, tsink, _query_ctx.get()),
+              Status::OK());
+
+    auto& local_state = _runtime_states.back()
+                                .front()
+                                ->get_local_state(op->operator_id())
+                                ->cast<ExchangeLocalState>();
+    auto& sink_local_state =
+            _runtime_states.back().front()->get_sink_local_state()->cast<ExchangeSinkLocalState>();
+    EXPECT_EQ(sink_local_state.channels.size(), 1);
+    EXPECT_EQ(sink_local_state.only_local_exchange, true);
+
+    EXPECT_EQ(local_state.stream_recvr->sender_queues().size(), 1);
+
+    // Blocked by execution dependency which is set by FE 2-phase trigger.
+    EXPECT_EQ(_pipeline_tasks[pipeline_id].back()->_wait_to_start(), true);
+    _query_ctx->get_execution_dependency()->set_ready();
+    // Task is ready and be push into runnable task queue.
+    EXPECT_EQ(_task_queue->take(0) != nullptr, true);
+    EXPECT_EQ(_pipeline_tasks[pipeline_id].back()->_wait_to_start(), false);
+
+    bool eos = false;
+    auto read_deps = local_state.dependencies();
+    {
+        // Blocked by exchange read dependency due to no data reached.
+        EXPECT_EQ(_pipeline_tasks[pipeline_id].back()->execute(&eos), Status::OK());
+        EXPECT_EQ(_pipeline_tasks[pipeline_id].back()->_opened, true);
+        EXPECT_EQ(_pipeline_tasks[pipeline_id].back()->_wake_up_early ||
+                          _pipeline_tasks[pipeline_id].back()->_eos || eos,
+                  false);
+    }
+    EXPECT_EQ(sink_local_state.channels[0]->_local_recvr != nullptr, true);
+    EXPECT_EQ(std::all_of(read_deps.cbegin(), read_deps.cend(),
+                          [](const auto& dep) { return dep->ready(); }),
+              false);
+
+    // Construct input block
+    vectorized::Block block;
+    {
+        vectorized::DataTypePtr int_type = std::make_shared<vectorized::DataTypeInt32>();
+
+        auto int_col0 = vectorized::ColumnInt32::create();
+        int_col0->insert_many_vals(1, 10);
+        block.insert({std::move(int_col0), int_type, "test_int_col0"});
+    }
+
+    local_state.stream_recvr->_sender_queues[0]->add_block(&block, true);
+    EXPECT_EQ(block.columns(), 0);
+
+    // Task is ready since a new block reached.
+    EXPECT_EQ(_task_queue->take(0) != nullptr, true);
+    EXPECT_EQ(std::all_of(read_deps.cbegin(), read_deps.cend(),
+                          [](const auto& dep) { return dep->ready(); }),
+              true);
+    {
+        // Blocked by exchange read dependency due to no data reached.
+        EXPECT_EQ(_pipeline_tasks[pipeline_id].back()->execute(&eos), Status::OK());
+        EXPECT_EQ(std::all_of(read_deps.cbegin(), read_deps.cend(),
+                              [](const auto& dep) { return dep->ready(); }),
+                  false);
+        EXPECT_EQ(_pipeline_tasks[pipeline_id].back()->_wake_up_early ||
+                          _pipeline_tasks[pipeline_id].back()->_eos || eos,
+                  false);
+    }
+    // Upstream task finished.
+    local_state.stream_recvr->_sender_queues[0]->decrement_senders(0);
+    EXPECT_EQ(_task_queue->take(0) != nullptr, true);
+    {
+        // Blocked by exchange read dependency due to no data reached.
+        EXPECT_EQ(_pipeline_tasks[pipeline_id].back()->execute(&eos), Status::OK());
+        EXPECT_EQ(std::all_of(read_deps.cbegin(), read_deps.cend(),
+                              [](const auto& dep) { return dep->ready(); }),
+                  true);
+        EXPECT_EQ(eos, true);
+        EXPECT_EQ(_pipeline_tasks[pipeline_id].back()->is_pending_finish(), false);
+        EXPECT_EQ(_pipeline_tasks[pipeline_id].back()->close(Status::OK()), Status::OK());
+    }
+    {
+        EXPECT_EQ(downstream_recvr->_sender_queues[0]->_block_queue.size(), 1);
+        EXPECT_EQ(downstream_recvr->_sender_queues[0]->_num_remaining_senders, 0);
+    }
+}
+
+} // namespace doris::pipeline
diff --git a/be/test/pipeline/thrift_builder.h b/be/test/pipeline/thrift_builder.h
new file mode 100644
index 000000000000000..0d255575179b071
--- /dev/null
+++ b/be/test/pipeline/thrift_builder.h
@@ -0,0 +1,354 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+#include <memory>
+#include <vector>
+
+#include "common/status.h"
+#include "pipeline/exec/exchange_sink_buffer.h"
+#include "pipeline/exec/exchange_sink_operator.h"
+#include "runtime/runtime_state.h"
+#include "udf/udf.h"
+#include "vec/sink/writer/vhive_utils.h"
+
+namespace doris::pipeline {
+
+class TQueryOptionsBuilder {
+public:
+    explicit TQueryOptionsBuilder() : _query_options() {}
+
+    TQueryOptionsBuilder& set_batch_size(int batch_size) {
+        _query_options.batch_size = batch_size;
+        return *this;
+    }
+    TQueryOptionsBuilder& set_execution_timeout(int execution_timeout) {
+        _query_options.execution_timeout = execution_timeout;
+        return *this;
+    }
+    TQueryOptionsBuilder& set_mem_limit(int64_t mem_limit) {
+        _query_options.mem_limit = mem_limit;
+        return *this;
+    }
+    TQueryOptionsBuilder& set_query_type(TQueryType::type& query_type) {
+        _query_options.query_type = query_type;
+        return *this;
+    }
+    TQueryOptionsBuilder& set_be_exec_version(int be_exec_version) {
+        _query_options.be_exec_version = be_exec_version;
+        return *this;
+    }
+    TQueryOptionsBuilder& set_enable_local_exchange(bool enable_local_exchange) {
+        _query_options.enable_local_exchange = enable_local_exchange;
+        return *this;
+    }
+    TQueryOptionsBuilder& set_enable_new_shuffle_hash_method(bool enable_new_shuffle_hash_method) {
+        _query_options.enable_new_shuffle_hash_method = enable_new_shuffle_hash_method;
+        return *this;
+    }
+    TQueryOptionsBuilder& set_enable_local_shuffle(bool enable_local_shuffle) {
+        _query_options.enable_local_shuffle = enable_local_shuffle;
+        return *this;
+    }
+    TQueryOptionsBuilder& set_runtime_filter_wait_infinitely(bool runtime_filter_wait_infinitely) {
+        _query_options.runtime_filter_wait_infinitely = runtime_filter_wait_infinitely;
+        return *this;
+    }
+    TQueryOptionsBuilder& set_enable_local_merge_sort(bool enable_local_merge_sort) {
+        _query_options.enable_local_merge_sort = enable_local_merge_sort;
+        return *this;
+    }
+
+    TQueryOptions& build() { return _query_options; }
+
+    TQueryOptionsBuilder(const TQueryOptionsBuilder&) = delete;
+    void operator=(const TQueryOptionsBuilder&) = delete;
+
+private:
+    TQueryOptions _query_options;
+};
+
+class TPlanNodeBuilder {
+public:
+    explicit TPlanNodeBuilder(TPlanNodeId node_id, TPlanNodeType::type node_type,
+                              int num_children = 0, int64_t limit = -1, bool compact_data = true)
+            : _plan_node() {
+        _plan_node.node_id = node_id;
+        _plan_node.node_type = node_type;
+        _plan_node.num_children = num_children;
+        _plan_node.limit = limit;
+        _plan_node.compact_data = compact_data;
+    }
+
+    TPlanNodeBuilder& set_is_serial_operator(bool is_serial_operator) {
+        _plan_node.is_serial_operator = is_serial_operator;
+        return *this;
+    }
+    TPlanNodeBuilder& set_exchange_node(TExchangeNode& node) {
+        _plan_node.exchange_node = node;
+        return *this;
+    }
+    TPlanNodeBuilder& append_row_tuples(TTupleId tuple_id, bool nullable) {
+        _plan_node.row_tuples.emplace_back(tuple_id);
+        _plan_node.nullable_tuples.emplace_back(nullable);
+        return *this;
+    }
+
+    TPlanNode& build() { return _plan_node; }
+
+    TPlanNodeBuilder(const TPlanNodeBuilder&) = delete;
+    void operator=(const TPlanNodeBuilder&) = delete;
+
+private:
+    TPlanNode _plan_node;
+};
+
+class TExchangeNodeBuilder {
+public:
+    explicit TExchangeNodeBuilder() : _plan_node() {}
+
+    TExchangeNodeBuilder& set_partition_type(TPartitionType::type partition_type) {
+        _plan_node.partition_type = partition_type;
+        return *this;
+    }
+    TExchangeNodeBuilder& append_input_row_tuples(TTupleId tuple_id) {
+        _plan_node.input_row_tuples.emplace_back(tuple_id);
+        return *this;
+    }
+    TExchangeNode& build() { return _plan_node; }
+    TExchangeNodeBuilder(const TExchangeNodeBuilder&) = delete;
+    void operator=(const TExchangeNodeBuilder&) = delete;
+
+private:
+    TExchangeNode _plan_node;
+};
+
+class TDataSinkBuilder {
+public:
+    explicit TDataSinkBuilder(TDataSinkType::type type) : _sink() { _sink.type = type; }
+
+    TDataSinkBuilder& set_stream_sink(TDataStreamSink& stream_sink) {
+        _sink.stream_sink = stream_sink;
+        return *this;
+    }
+    TDataSink& build() { return _sink; }
+
+    TDataSinkBuilder(const TDataSinkBuilder&) = delete;
+    void operator=(const TDataSinkBuilder&) = delete;
+
+private:
+    TDataSink _sink;
+};
+
+class TDataStreamSinkBuilder {
+public:
+    explicit TDataStreamSinkBuilder(TPlanNodeId dest_node_id, TDataPartition output_partition)
+            : _sink() {
+        _sink.dest_node_id = dest_node_id;
+        _sink.output_partition = output_partition;
+    }
+
+    TDataStreamSink& build() { return _sink; }
+
+    TDataStreamSinkBuilder(const TDataStreamSinkBuilder&) = delete;
+    void operator=(const TDataStreamSinkBuilder&) = delete;
+
+private:
+    TDataStreamSink _sink;
+};
+
+class TPlanFragmentDestinationBuilder {
+public:
+    explicit TPlanFragmentDestinationBuilder(TUniqueId fragment_instance_id, TNetworkAddress server,
+                                             TNetworkAddress brpc_server)
+            : _dest() {
+        _dest.fragment_instance_id = fragment_instance_id;
+        _dest.server = server;
+        _dest.brpc_server = brpc_server;
+    }
+
+    TPlanFragmentDestination& build() { return _dest; }
+
+    TPlanFragmentDestinationBuilder(const TPlanFragmentDestinationBuilder&) = delete;
+    void operator=(const TPlanFragmentDestinationBuilder&) = delete;
+
+private:
+    TPlanFragmentDestination _dest;
+};
+
+class TTupleDescriptorBuilder {
+public:
+    explicit TTupleDescriptorBuilder() : _desc() {
+        _desc.byteSize = 0;
+        _desc.numNullBytes = 0;
+    }
+
+    TTupleDescriptorBuilder& set_id(TTupleId id) {
+        _desc.id = id;
+        return *this;
+    }
+
+    TTupleDescriptor& build() { return _desc; }
+    TTupleDescriptorBuilder(const TTupleDescriptorBuilder&) = delete;
+    void operator=(const TTupleDescriptorBuilder&) = delete;
+
+private:
+    TTupleDescriptor _desc;
+};
+
+class TSlotDescriptorBuilder {
+public:
+    explicit TSlotDescriptorBuilder() : _desc() {
+        _desc.columnPos = -1;
+        _desc.nullIndicatorByte = 0;
+    }
+    TSlotDescriptorBuilder& set_id(TTupleId id) {
+        _desc.id = id;
+        return *this;
+    }
+    TSlotDescriptorBuilder& set_parent(TTupleDescriptor& parent) {
+        _desc.parent = parent.id;
+        return *this;
+    }
+    TSlotDescriptorBuilder& set_slotType(TTypeDesc& slotType) {
+        _desc.slotType = slotType;
+        return *this;
+    }
+    TSlotDescriptorBuilder& set_nullIndicatorBit(int nullIndicatorBit) {
+        _desc.nullIndicatorBit = nullIndicatorBit;
+        return *this;
+    }
+    TSlotDescriptorBuilder& set_byteOffset(int byteOffset) {
+        _desc.byteOffset = byteOffset;
+        return *this;
+    }
+    TSlotDescriptorBuilder& set_slotIdx(int slotIdx) {
+        _desc.slotIdx = slotIdx;
+        return *this;
+    }
+    TSlotDescriptorBuilder& set_isMaterialized(bool isMaterialized) {
+        _desc.isMaterialized = isMaterialized;
+        return *this;
+    }
+    TSlotDescriptorBuilder& set_colName(std::string colName) {
+        _desc.colName = colName;
+        return *this;
+    }
+
+    TSlotDescriptor& build() { return _desc; }
+    TSlotDescriptorBuilder(const TSlotDescriptorBuilder&) = delete;
+    void operator=(const TSlotDescriptorBuilder&) = delete;
+
+private:
+    TSlotDescriptor _desc;
+};
+
+class TTypeDescBuilder {
+public:
+    explicit TTypeDescBuilder() : _desc() {}
+
+    TTypeDescBuilder& set_types(TTypeNode type_node) {
+        _desc.types.push_back(type_node);
+        return *this;
+    }
+
+    TTypeDesc& build() { return _desc; }
+    TTypeDescBuilder(const TTypeDescBuilder&) = delete;
+    void operator=(const TTypeDescBuilder&) = delete;
+
+private:
+    TTypeDesc _desc;
+};
+
+class TTypeNodeBuilder {
+public:
+    explicit TTypeNodeBuilder() : _desc() {}
+
+    TTypeNodeBuilder& set_type(TTypeNodeType::type type) {
+        _desc.type = type;
+        return *this;
+    }
+
+    TTypeNodeBuilder& set_scalar_type(TPrimitiveType::type type, int len = 0, int precision = 0,
+                                      int scale = 0) {
+        TScalarType scalar_type;
+        scalar_type.type = type;
+        scalar_type.__set_len(len);
+        scalar_type.__set_precision(precision);
+        scalar_type.__set_scale(scale);
+        _desc.__set_scalar_type(scalar_type);
+        return *this;
+    }
+
+    TTypeNode& build() { return _desc; }
+    TTypeNodeBuilder(const TTypeNodeBuilder&) = delete;
+    void operator=(const TTypeNodeBuilder&) = delete;
+
+private:
+    TTypeNode _desc;
+};
+
+class TDescriptorTableBuilder {
+public:
+    explicit TDescriptorTableBuilder() : _desc() {}
+
+    TDescriptorTableBuilder& append_slotDescriptors(TSlotDescriptor& desc) {
+        _desc.slotDescriptors.push_back(desc);
+        return *this;
+    }
+    TDescriptorTableBuilder& append_tupleDescriptors(TTupleDescriptor& desc) {
+        _desc.tupleDescriptors.push_back(desc);
+        return *this;
+    }
+    TDescriptorTableBuilder& append_tableDescriptors(TTableDescriptor& desc) {
+        _desc.tableDescriptors.push_back(desc);
+        return *this;
+    }
+
+    TDescriptorTable& build() { return _desc; }
+    TDescriptorTableBuilder(const TDescriptorTableBuilder&) = delete;
+    void operator=(const TDescriptorTableBuilder&) = delete;
+
+private:
+    TDescriptorTable _desc;
+};
+
+class TDataPartitionBuilder {
+public:
+    explicit TDataPartitionBuilder(TPartitionType::type type) : _partition() {
+        _partition.type = type;
+    }
+
+    TDataPartitionBuilder& append_partition_exprs(TExpr expr) {
+        _partition.partition_exprs.push_back(expr);
+        return *this;
+    }
+    TDataPartitionBuilder& append_partition_infos(TRangePartition info) {
+        _partition.partition_infos.push_back(info);
+        return *this;
+    }
+
+    TDataPartition& build() { return _partition; }
+    TDataPartitionBuilder(const TDataPartitionBuilder&) = delete;
+    void operator=(const TDataPartitionBuilder&) = delete;
+
+private:
+    TDataPartition _partition;
+};
+
+} // namespace doris::pipeline

From e48f3ee30f975d5d73c8fe3e8b00bd01bf4dfff3 Mon Sep 17 00:00:00 2001
From: Pxl <xl@selectdb.com>
Date: Mon, 13 Jan 2025 10:19:03 +0800
Subject: [PATCH 130/140] [Chore](schema-change) return error when
 tablet_schema get null function (#46790)

### What problem does this PR solve?

return error when tablet_schema get null function

part of #46631
---
 be/src/olap/schema_change.cpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index 4a737a1316e0cd8..91978df30fd02cf 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -137,6 +137,12 @@ class MultiBlockMerger {
                             tablet_schema->column(i).get_aggregate_function(
                                     vectorized::AGG_LOAD_SUFFIX,
                                     tablet_schema->column(i).get_be_exec_version());
+                    if (!function) {
+                        return Status::InternalError(
+                                "could not find aggregate function on column {}, aggregation={}",
+                                tablet_schema->column(i).name(),
+                                tablet_schema->column(i).aggregation());
+                    }
                     agg_functions.push_back(function);
                     // create aggregate data
                     auto* place = new char[function->size_of_data()];

From d6958498e14c9d20eb0be9b2a6e962896b90c64a Mon Sep 17 00:00:00 2001
From: zclllyybb <zhaochangle@selectdb.com>
Date: Mon, 13 Jan 2025 10:24:33 +0800
Subject: [PATCH 131/140] [Fix](session variable) Forward use_fix_replica to
 master (#46836)

---
 .../src/main/java/org/apache/doris/qe/SessionVariable.java      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 8f35ef4cb9da69b..a52beff44bb73ed 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -1714,7 +1714,7 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) {
     public boolean showUserDefaultRole = false;
 
     // Default value is -1, which means not fix replica
-    @VariableMgr.VarAttr(name = USE_FIX_REPLICA)
+    @VariableMgr.VarAttr(name = USE_FIX_REPLICA, needForward = true)
     public int useFixReplica = -1;
 
     @VariableMgr.VarAttr(name = DUMP_NEREIDS_MEMO)

From 87858e58d86c3cde6247806bf98e8e188e703eef Mon Sep 17 00:00:00 2001
From: zhiqiang <hezhiqiang@selectdb.com>
Date: Mon, 13 Jan 2025 10:36:28 +0800
Subject: [PATCH 132/140] [fix](regression test) Fix case
 test_show_commands_nereids (#46846)

---
 .../nereids_p0/show/test_show_commands_nereids.groovy  | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/regression-test/suites/nereids_p0/show/test_show_commands_nereids.groovy b/regression-test/suites/nereids_p0/show/test_show_commands_nereids.groovy
index 87a7087d9956e4e..52b3383deb479fb 100644
--- a/regression-test/suites/nereids_p0/show/test_show_commands_nereids.groovy
+++ b/regression-test/suites/nereids_p0/show/test_show_commands_nereids.groovy
@@ -27,12 +27,6 @@ suite("test_show_commands_nereids") {
     checkNereidsExecute("""show whitelist;""")
     checkNereidsExecute("""show triggers;""")
     checkNereidsExecute("""show events;""")
-    test {
-        sql """show load profile "/";"""
-        exception """show query/load profile syntax is a deprecated feature"""
-    }
-    test {
-        sql """show query profile "/";"""
-        exception """show query/load profile syntax is a deprecated feature"""
-    }
+    checkNereidsExecute("""show load profile "/";""")
+    checkNereidsExecute("""show query profile "/";""")    
 }

From d9eb14a96bbe0cda1a6335c67bf3e9c4ef30513b Mon Sep 17 00:00:00 2001
From: deardeng <dengxin@selectdb.com>
Date: Mon, 13 Jan 2025 11:10:04 +0800
Subject: [PATCH 133/140] [fix](cloud) Fix cloud decomission lead to fe cant
 start (#46783)

Fix issue with SQL node decommissioning process

The SQL node decommissioning process does not wait for transactions at
the watermark level to complete before setting the backend's
isDecommissioned status to true.

As a result, the value displayed in show backends immediately reflects
isDecommissioned regardless of ongoing transactions initiated via SQL.

When a user calls drop be to remove a backend while there is only one
backend in the cluster, the edit log logs the drop backend action, which
removes the cluster information from memory.

After dropping the backend, the previous transaction watermark process
completes its tasks and attempts to modify the backend status, which
requires accessing the cluster information. However, since the cluster
information has already been deleted, this results in a null pointer
exception (NPE) during the lookup in the FE memory map, causing the FE
to crash.

Additionally, the sequence of edit logs is fixed as follows:

Edit log logs drop backend
Edit log modifies backend
FE fails to start up


```
2025-01-10 05:46:15,070 ERROR (replayer|15) [EditLog.loadJournal():1251] replay Operation Type 91, log id: 10578
java.lang.NullPointerException: Cannot invoke "org.apache.doris.system.Backend.getCloudClusterName()" because "memBe" is null
        at org.apache.doris.cloud.system.CloudSystemInfoService.replayModifyBackend(CloudSystemInfoService.java:461) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.persist.EditLog.loadJournal(EditLog.java:432) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.catalog.Env.replayJournal(Env.java:2999) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.catalog.Env$4.runOneCycle(Env.java:2761) ~[doris-fe.jar:1.2-SNAPSHOT]
        at org.apache.doris.common.util.Daemon.run(Daemon.java:119) ~[doris-fe.jar:1.2-SNAPSHOT]
```
---
 .../org/apache/doris/cloud/system/CloudSystemInfoService.java  | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java
index 36ca260dc17f725..71260c51f239139 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java
@@ -457,6 +457,9 @@ public void replayDropBackend(Backend backend) {
     @Override
     public void replayModifyBackend(Backend backend) {
         Backend memBe = getBackend(backend.getId());
+        if (memBe == null) {
+            return;
+        }
         // for rename cluster
         String originalClusterName = memBe.getCloudClusterName();
         String originalClusterId = memBe.getCloudClusterId();

From be2932accb68b20a65f0404793119f415deaa31d Mon Sep 17 00:00:00 2001
From: deardeng <dengxin@selectdb.com>
Date: Mon, 13 Jan 2025 11:25:50 +0800
Subject: [PATCH 134/140] [fix](cloud) Support clean tablet file cache when
 tablet drop (#46390)

---
 be/src/agent/task_worker_pool.cpp             |  43 +++++-
 be/src/cloud/cloud_tablet.cpp                 |   6 +-
 be/src/cloud/cloud_tablet.h                   |   4 +-
 .../doris/regression/suite/Suite.groovy       |  91 ++++++++++++
 .../apache/doris/regression/util/Http.groovy  |   2 +-
 .../test_clean_stale_rs_file_cache.groovy     | 129 +++++++++++++++++
 ..._clean_tablet_when_drop_force_table.groovy | 132 ++++++++++++------
 .../test_clean_tablet_when_rebalance.groovy   | 123 +++++++++-------
 8 files changed, 424 insertions(+), 106 deletions(-)
 create mode 100644 regression-test/suites/cloud_p0/tablets/test_clean_stale_rs_file_cache.groovy

diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp
index 40b84facecfc5d1..329d94eaf3869a7 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -1659,11 +1659,46 @@ void drop_tablet_callback(CloudStorageEngine& engine, const TAgentTaskRequest& r
                 .tag("tablet_id", drop_tablet_req.tablet_id);
         return;
     });
-    // 1. erase lru from tablet mgr
-    // TODO(dx) clean tablet file cache
-    // get tablet's info(such as cachekey, tablet id, rsid)
+    MonotonicStopWatch watch;
+    watch.start();
+    auto weak_tablets = engine.tablet_mgr().get_weak_tablets();
+    std::ostringstream rowset_ids_stream;
+    bool found = false;
+    for (auto& weak_tablet : weak_tablets) {
+        auto tablet = weak_tablet.lock();
+        if (tablet == nullptr) {
+            continue;
+        }
+        if (tablet->tablet_id() != drop_tablet_req.tablet_id) {
+            continue;
+        }
+        found = true;
+        auto clean_rowsets = tablet->get_snapshot_rowset(true);
+        // Get first 10 rowset IDs as comma-separated string, just for log
+        int count = 0;
+        for (const auto& rowset : clean_rowsets) {
+            if (count >= 10) break;
+            if (count > 0) {
+                rowset_ids_stream << ",";
+            }
+            rowset_ids_stream << rowset->rowset_id().to_string();
+            count++;
+        }
+
+        CloudTablet::recycle_cached_data(std::move(clean_rowsets));
+        break;
+    }
+
+    if (!found) {
+        LOG(WARNING) << "tablet not found when dropping tablet_id=" << drop_tablet_req.tablet_id
+                     << ", cost " << static_cast<int64_t>(watch.elapsed_time()) / 1e9 << "(s)";
+        return;
+    }
+
     engine.tablet_mgr().erase_tablet(drop_tablet_req.tablet_id);
-    // 2. gen clean file cache task
+    LOG(INFO) << "drop cloud tablet_id=" << drop_tablet_req.tablet_id
+              << " and clean file cache first 10 rowsets {" << rowset_ids_stream.str() << "}, cost "
+              << static_cast<int64_t>(watch.elapsed_time()) / 1e9 << "(s)";
     return;
 }
 
diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp
index f2f21162bf05b55..b8dd0eae4b736ef 100644
--- a/be/src/cloud/cloud_tablet.cpp
+++ b/be/src/cloud/cloud_tablet.cpp
@@ -386,6 +386,8 @@ void CloudTablet::delete_rowsets(const std::vector<RowsetSharedPtr>& to_delete,
 
 uint64_t CloudTablet::delete_expired_stale_rowsets() {
     std::vector<RowsetSharedPtr> expired_rowsets;
+    // ATTN: trick, Use stale_rowsets to temporarily increase the reference count of the rowset shared pointer in _stale_rs_version_map so that in the recycle_cached_data function, it checks if the reference count is 2.
+    std::vector<RowsetSharedPtr> stale_rowsets;
     int64_t expired_stale_sweep_endtime =
             ::time(nullptr) - config::tablet_rowset_stale_sweep_time_sec;
     std::vector<std::string> version_to_delete;
@@ -409,6 +411,7 @@ uint64_t CloudTablet::delete_expired_stale_rowsets() {
                 auto rs_it = _stale_rs_version_map.find(v_ts->version());
                 if (rs_it != _stale_rs_version_map.end()) {
                     expired_rowsets.push_back(rs_it->second);
+                    stale_rowsets.push_back(rs_it->second);
                     LOG(INFO) << "erase stale rowset, tablet_id=" << tablet_id()
                               << " rowset_id=" << rs_it->second->rowset_id().to_string()
                               << " version=" << rs_it->first.to_string();
@@ -456,7 +459,8 @@ void CloudTablet::recycle_cached_data(const std::vector<RowsetSharedPtr>& rowset
 
     if (config::enable_file_cache) {
         for (const auto& rs : rowsets) {
-            if (rs.use_count() >= 1) {
+            // rowsets and tablet._rs_version_map each hold a rowset shared_ptr, so at this point, the reference count of the shared_ptr is at least 2.
+            if (rs.use_count() > 2) {
                 LOG(WARNING) << "Rowset " << rs->rowset_id().to_string() << " has "
                              << rs.use_count()
                              << " references. File Cache won't be recycled when query is using it.";
diff --git a/be/src/cloud/cloud_tablet.h b/be/src/cloud/cloud_tablet.h
index c876518d868a497..2aa031ad503f431 100644
--- a/be/src/cloud/cloud_tablet.h
+++ b/be/src/cloud/cloud_tablet.h
@@ -210,12 +210,12 @@ class CloudTablet final : public BaseTablet {
 
     void build_tablet_report_info(TTabletInfo* tablet_info);
 
+    static void recycle_cached_data(const std::vector<RowsetSharedPtr>& rowsets);
+
 private:
     // FIXME(plat1ko): No need to record base size if rowsets are ordered by version
     void update_base_size(const Rowset& rs);
 
-    static void recycle_cached_data(const std::vector<RowsetSharedPtr>& rowsets);
-
     Status sync_if_not_running();
 
     CloudStorageEngine& _engine;
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index 5887fd607d44a91..2b121cb8f4497d4 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -49,6 +49,7 @@ import org.apache.doris.regression.action.HttpCliAction
 import org.apache.doris.regression.util.DataUtils
 import org.apache.doris.regression.util.JdbcUtils
 import org.apache.doris.regression.util.Hdfs
+import org.apache.doris.regression.util.Http
 import org.apache.doris.regression.util.SuiteUtils
 import org.apache.doris.regression.util.DebugPoint
 import org.apache.doris.regression.RunMode
@@ -2831,4 +2832,94 @@ class Suite implements GroovyInterceptable {
         assertEquals(re_fe, re_be)
         assertEquals(re_fe, re_no_fold)
     }
+
+    def backendIdToHost = { ->
+        def spb = sql_return_maparray """SHOW BACKENDS"""
+        def beIdToHost = [:]
+        spb.each {
+            beIdToHost[it.BackendId] = it.Host
+        }
+        beIdToHost 
+    }
+
+    def getTabletAndBeHostFromBe = { bes ->
+        def ret = [:]
+        bes.each { be ->
+            // {"msg":"OK","code":0,"data":{"host":"128.2.51.2","tablets":[{"tablet_id":10560},{"tablet_id":10554},{"tablet_id":10552}]},"count":3}
+            def data = Http.GET("http://${be.host}:${be.httpPort}/tablets_json?limit=all", true).data
+            def tablets = data.tablets.collect { it.tablet_id as String }
+            tablets.each{
+                ret[it] = data.host
+            }
+        }
+        ret
+    }
+
+    def getTabletAndBeHostFromFe = { table ->
+        def result = sql_return_maparray """SHOW TABLETS FROM $table"""
+        def bes = backendIdToHost.call()
+        // tablet : [backendId, host]
+        def ret = [:]
+        result.each {
+            ret[it.TabletId] = [it.BackendId, bes[it.BackendId]]
+        }
+        ret
+    }
+
+    // get rowset_id segment_id from ms
+    // curl '175.40.101.1:5000/MetaService/http/get_value?token=greedisgood9999&unicode&key_type=MetaRowsetKey&instance_id=default_instance_id&tablet_id=27700&version=2'
+    def getSegmentFilesFromMs = { msHttpPort, tabletId, version, check_func ->
+        httpTest {
+            endpoint msHttpPort
+            op "get"
+            uri "/MetaService/http/get_value?token=greedisgood9999&unicode&key_type=MetaRowsetKey&instance_id=default_instance_id&tablet_id=${tabletId}&version=${version}"
+            check check_func
+        }
+    }
+
+    def getRowsetFileCacheDirFromBe = { beHttpPort, msHttpPort, tabletId, version -> 
+        def hashValues = []
+        def segmentFiles = []
+        getSegmentFilesFromMs(msHttpPort, tabletId, version) {
+            respCode, body ->
+                def json = parseJson(body)
+                logger.info("get tablet {} version {} from ms, response {}", tabletId, version, json)
+                // {"rowset_id":"0","partition_id":"27695","tablet_id":"27700","txn_id":"7057526525952","tablet_schema_hash":0,"rowset_type":"BETA_ROWSET","rowset_state":"COMMITTED","start_version":"3","end_version":"3","version_hash":"0","num_rows":"1","total_disk_size":"895","data_disk_size":"895","index_disk_size":"0","empty":false,"load_id":{"hi":"-1646598626735601581","lo":"-6677682539881484579"},"delete_flag":false,"creation_time":"1736153402","num_segments":"1","rowset_id_v2":"0200000000000004694889e84c76391cfd52ec7db0a483ba","resource_id":"1","newest_write_timestamp":"1736153402","segments_key_bounds":[{"min_key":"AoAAAAAAAAAC","max_key":"AoAAAAAAAAAC"}],"txn_expiration":"1736167802","segments_overlap_pb":"NONOVERLAPPING","compaction_level":"0","segments_file_size":["895"],"index_id":"27697","schema_version":0,"enable_segments_file_size":true,"has_variant_type_in_schema":false,"enable_inverted_index_file_info":false}
+                def segmentNum = json.num_segments as int
+                def rowsetId = json.rowset_id_v2 as String
+                segmentFiles = (0..<segmentNum).collect { i -> "${rowsetId}_${i}.dat" }
+        }
+
+        segmentFiles.each {
+            // curl '175.40.51.3:8040/api/file_cache?op=hash&value=0200000000000004694889e84c76391cfd52ec7db0a483ba_0.dat'
+            def data = Http.GET("http://${beHttpPort}/api/file_cache?op=hash&value=${it}", true)
+            // {"hash":"2b79c649a1766dad371054ee168f0574"}
+            logger.info("get tablet {} segmentFile {}, response {}", tabletId, it, data)
+            hashValues << data.hash
+        }
+        hashValues
+    }
+
+    // get table's tablet file cache
+    def getTabletFileCacheDirFromBe = { msHttpPort, table, version ->
+        // beHost HashFile
+        def beHostToHashFile = [:]
+
+        def getTabletsAndHostFromFe = getTabletAndBeHostFromFe(table)
+        getTabletsAndHostFromFe.each {
+            def beHost = it.Value[1]
+            def tabletId = it.Key
+            def hashRet = getRowsetFileCacheDirFromBe(beHost + ":8040", msHttpPort, tabletId, version)
+            hashRet.each {
+                def hashFile = it
+                if (beHostToHashFile.containsKey(beHost)) {
+                    beHostToHashFile[beHost].add(hashFile)
+                } else {
+                    beHostToHashFile[beHost] = [hashFile]
+                }
+            }
+        }
+        beHostToHashFile
+    }
+
 }
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/Http.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/Http.groovy
index cd688a1fcfc0624..2a63f8763df80fd 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/Http.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/Http.groovy
@@ -53,7 +53,7 @@ class Http {
         conn.setRequestProperty('Authorization', 'Basic cm9vdDo=') //token for root
         def code = conn.responseCode
         def text = conn.content.text
-        logger.info("http post url=${url}, isJson=${isJson}, response code=${code}, text=${text}")
+        logger.info("http get url=${url}, isJson=${isJson}, response code=${code}, text=${text}")
         Assert.assertEquals(200, code)
         if (isJson) {
             def json = new JsonSlurper()
diff --git a/regression-test/suites/cloud_p0/tablets/test_clean_stale_rs_file_cache.groovy b/regression-test/suites/cloud_p0/tablets/test_clean_stale_rs_file_cache.groovy
new file mode 100644
index 000000000000000..8d41939981a812e
--- /dev/null
+++ b/regression-test/suites/cloud_p0/tablets/test_clean_stale_rs_file_cache.groovy
@@ -0,0 +1,129 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.doris.regression.suite.ClusterOptions
+import org.apache.doris.regression.util.Http
+
+suite('test_clean_stale_rs_file_cache', 'docker') {
+    if (!isCloudMode()) {
+        return;
+    }
+    def options = new ClusterOptions()
+    options.feConfigs += [
+        'cloud_cluster_check_interval_second=1',
+        'cloud_tablet_rebalancer_interval_second=1',
+        'sys_log_verbose_modules=org',
+        'heartbeat_interval_second=1'
+    ]
+    options.beConfigs += [
+        'report_tablet_interval_seconds=1',
+        'cumulative_compaction_min_deltas=5',
+        'tablet_rowset_stale_sweep_by_size=false',
+        'tablet_rowset_stale_sweep_time_sec=60',
+        'vacuum_stale_rowsets_interval_s=10'
+    ]
+    options.setFeNum(1)
+    options.setBeNum(1)
+    options.cloudMode = true
+
+    def table = "test_clean_stale_rs_file_cache"
+
+    docker(options) {
+        def ms = cluster.getAllMetaservices().get(0)
+        def msHttpPort = ms.host + ":" + ms.httpPort
+        sql """CREATE TABLE $table (
+            `k1` int(11) NULL,
+            `k2` int(11) NULL,
+            `v1` varchar(2048)
+            )
+            DUPLICATE KEY(`k1`, `k2`)
+            COMMENT 'OLAP'
+            DISTRIBUTED BY HASH(`k1`) BUCKETS 1
+            PROPERTIES (
+            "replication_num"="1"
+            );
+        """
+        // version 2
+        sql """
+            insert into $table values (1, 1, 'v1'), (2, 2, 'v2'), (3, 3, 'v3')
+        """
+        def cacheDirVersion2 = getTabletFileCacheDirFromBe(msHttpPort, table, 2)
+        // version 3
+        sql """
+            insert into $table values (10, 1, 'v1'), (20, 2, 'v2'), (30, 3, 'v3')
+        """
+        def cacheDirVersion3 = getTabletFileCacheDirFromBe(msHttpPort, table, 2)
+        // version 4
+        sql """
+            insert into $table values (100, 1, 'v1'), (200, 2, 'v2'), (300, 3, 'v3')
+        """
+        // version 5
+        sql """
+            insert into $table values (1000, 1, 'v1'), (2000, 2, 'v2'), (3000, 3, 'v3')
+        """
+        // version 6
+        sql """
+            insert into $table values (10000, 1, 'v1'), (20000, 2, 'v2'), (30000, 3, 'v3')
+        """
+
+        def mergedCacheDir = cacheDirVersion2 + cacheDirVersion3.collectEntries { host, hashFiles ->
+            [(host): cacheDirVersion2[host] ? (cacheDirVersion2[host] + hashFiles) : hashFiles]
+        }
+        for (int i = 0; i < 5; i++) {
+            sql """
+                select count(*) from $table
+            """
+        }
+        def beforeGetFromFe = getTabletAndBeHostFromFe(table)
+        logger.info("fe tablets {}, cache dir {}", beforeGetFromFe , mergedCacheDir)
+        // wait compaction finish, and vacuum_stale_rowsets work
+        sleep(80 * 1000)
+
+        // check cache file has been deleted
+        beforeGetFromFe.each {
+            def tabletId = it.Key
+            def backendId = it.Value[0]
+            def backendHost = it.Value[1]
+            def be = cluster.getBeByBackendId(backendId.toLong())
+            def dataPath = new File("${be.path}/storage/file_cache")
+            def subDirs = []
+            
+            def collectDirs
+            collectDirs = { File dir ->
+                if (dir.exists()) {
+                    dir.eachDir { subDir ->
+                        subDirs << subDir.name
+                        collectDirs(subDir) 
+                    }
+                }
+            }
+            
+            
+            collectDirs(dataPath)
+            logger.info("BE {} file_cache subdirs: {}", backendHost, subDirs)
+            def cacheDir = mergedCacheDir[backendHost]
+
+            // add check
+            cacheDir.each { hashFile ->
+                assertFalse(subDirs.any { subDir -> subDir.startsWith(hashFile) }, 
+                "Found unexpected cache file pattern ${hashFile} in BE ${backendHost}'s file_cache directory. " + 
+                "Matching subdir found in: ${subDirs}")
+            }
+        }
+
+    }
+}
diff --git a/regression-test/suites/cloud_p0/tablets/test_clean_tablet_when_drop_force_table.groovy b/regression-test/suites/cloud_p0/tablets/test_clean_tablet_when_drop_force_table.groovy
index 4dc847d603a324c..a65f59f85a1ee41 100644
--- a/regression-test/suites/cloud_p0/tablets/test_clean_tablet_when_drop_force_table.groovy
+++ b/regression-test/suites/cloud_p0/tablets/test_clean_tablet_when_drop_force_table.groovy
@@ -16,7 +16,7 @@
 // under the License.
 
 import org.apache.doris.regression.suite.ClusterOptions
-import org.apache.doris.regression.util.Http
+
 
 suite('test_clean_tablet_when_drop_force_table', 'docker') {
     if (!isCloudMode()) {
@@ -31,51 +31,22 @@ suite('test_clean_tablet_when_drop_force_table', 'docker') {
         'rehash_tablet_after_be_dead_seconds=5'
     ]
     options.beConfigs += [
-        'report_tablet_interval_seconds=1'
+        'report_tablet_interval_seconds=1',
+        'write_buffer_size=10240',
+        'write_buffer_size_for_agg=10240'
     ]
     options.setFeNum(3)
     options.setBeNum(3)
     options.cloudMode = true
     options.enableDebugPoints()
     
-    def backendIdToHost = { ->
-        def spb = sql_return_maparray """SHOW BACKENDS"""
-        def beIdToHost = [:]
-        spb.each {
-            beIdToHost[it.BackendId] = it.Host
-        }
-        beIdToHost 
-    }
-
-    def getTabletAndBeHostFromFe = { table ->
-        def result = sql_return_maparray """SHOW TABLETS FROM $table"""
-        def bes = backendIdToHost.call()
-        // tablet : host
-        def ret = [:]
-        result.each {
-            ret[it.TabletId] = bes[it.BackendId]
-        }
-        ret
-    }
-
-    def getTabletAndBeHostFromBe = { ->
-        def bes = cluster.getAllBackends()
-        def ret = [:]
-        bes.each { be ->
-            // {"msg":"OK","code":0,"data":{"host":"128.2.51.2","tablets":[{"tablet_id":10560},{"tablet_id":10554},{"tablet_id":10552}]},"count":3}
-            def data = Http.GET("http://${be.host}:${be.httpPort}/tablets_json?limit=all", true).data
-            def tablets = data.tablets.collect { it.tablet_id as String }
-            tablets.each{
-                ret[it] = data.host
-            }
-        }
-        ret
-    }
-
     def testCase = { table, waitTime, useDp=false-> 
+        def ms = cluster.getAllMetaservices().get(0)
+        def msHttpPort = ms.host + ":" + ms.httpPort
         sql """CREATE TABLE $table (
             `k1` int(11) NULL,
-            `k2` int(11) NULL
+            `k2` int(11) NULL,
+            `v1` VARCHAR(2048)
             )
             DUPLICATE KEY(`k1`, `k2`)
             COMMENT 'OLAP'
@@ -84,23 +55,60 @@ suite('test_clean_tablet_when_drop_force_table', 'docker') {
             "replication_num"="1"
             );
         """
+        def random = new Random()
+        def generateRandomString = { int length ->
+            random.with {
+                def chars = ('A'..'Z').collect() + ('a'..'z').collect() + ('0'..'9').collect()
+                (1..length).collect { chars[nextInt(chars.size())] }.join('')
+            }
+        }
+        def valuesList = (1..30000).collect { i -> 
+            def randomStr = generateRandomString(2000)
+            "($i, $i, '$randomStr')"
+        }.join(", ")
+        sql """
+            set global max_allowed_packet = 1010241024
+        """
+
+        context.reconnectFe()
         sql """
-            insert into $table values (1, 1), (2, 2), (3, 3)
+            insert into $table values ${valuesList}
         """
 
         for (int i = 0; i < 5; i++) {
             sql """
-                select * from $table
+                select count(*) from $table
             """
         }
 
+        valuesList = (30001..60000).collect { i -> 
+            def randomStr = generateRandomString(2000)
+            "($i, $i, '$randomStr')"
+        }.join(", ")
+        sql """
+            set global max_allowed_packet = 1010241024
+        """
+        context.reconnectFe()
+        sql """
+            insert into $table values ${valuesList}
+        """
+
         // before drop table force
         def beforeGetFromFe = getTabletAndBeHostFromFe(table)
-        def beforeGetFromBe = getTabletAndBeHostFromBe.call()
-        logger.info("fe tablets {}, be tablets {}", beforeGetFromFe, beforeGetFromBe)
+        def beforeGetFromBe = getTabletAndBeHostFromBe(cluster.getAllBackends())
+        // version 2
+        def cacheDirVersion2 = getTabletFileCacheDirFromBe(msHttpPort, table, 2)
+        // version 3
+        def cacheDirVersion3 = getTabletFileCacheDirFromBe(msHttpPort, table, 3)
+
+        def mergedCacheDir = cacheDirVersion2 + cacheDirVersion3.collectEntries { host, hashFiles ->
+            [(host): cacheDirVersion2[host] ? (cacheDirVersion2[host] + hashFiles) : hashFiles]
+        }
+        
+        logger.info("fe tablets {}, be tablets {}, cache dir {}", beforeGetFromFe, beforeGetFromBe, mergedCacheDir)
         beforeGetFromFe.each {
             assertTrue(beforeGetFromBe.containsKey(it.Key))
-            assertEquals(beforeGetFromBe[it.Key], it.Value)
+            assertEquals(beforeGetFromBe[it.Key], it.Value[1])
         }
         if (useDp) {
             GetDebugPoint().enableDebugPointForAllBEs("WorkPoolCloudDropTablet.drop_tablet_callback.failed")
@@ -119,16 +127,50 @@ suite('test_clean_tablet_when_drop_force_table', 'docker') {
         }
         def start = System.currentTimeMillis() / 1000
         // tablet can't find in be 
-        dockerAwaitUntil(50) {
-            def beTablets = getTabletAndBeHostFromBe.call().keySet()
+        dockerAwaitUntil(500) {
+            def beTablets = getTabletAndBeHostFromBe(cluster.getAllBackends()).keySet()
             logger.info("before drop tablets {}, after tablets {}", beforeGetFromFe, beTablets)
-            beforeGetFromFe.keySet().every { !getTabletAndBeHostFromBe.call().containsKey(it) }
+            beforeGetFromFe.keySet().every { !getTabletAndBeHostFromBe(cluster.getAllBackends()).containsKey(it) }
         }
         logger.info("table {}, cost {}s", table, System.currentTimeMillis() / 1000 - start)
         assertTrue(System.currentTimeMillis() / 1000 - start > waitTime)
         if (useDp) {
             futrue.get()
         }
+
+        sleep(25 * 1000)
+
+        // check cache file has been deleted
+        beforeGetFromFe.each {
+            def tabletId = it.Key
+            def backendId = it.Value[0]
+            def backendHost = it.Value[1]
+            def be = cluster.getBeByBackendId(backendId.toLong())
+            def dataPath = new File("${be.path}/storage/file_cache")
+            def subDirs = []
+            
+            def collectDirs
+            collectDirs = { File dir ->
+                if (dir.exists()) {
+                    dir.eachDir { subDir ->
+                        subDirs << subDir.name
+                        collectDirs(subDir) 
+                    }
+                }
+            }
+            
+            collectDirs(dataPath)
+            logger.info("BE {} file_cache subdirs: {}", backendHost, subDirs)
+            def cacheDir = mergedCacheDir[backendHost]
+
+            // add check
+            cacheDir.each { hashFile ->
+                assertFalse(subDirs.any { subDir -> subDir.startsWith(hashFile) }, 
+                "Found unexpected cache file pattern ${hashFile} in BE ${backendHost}'s file_cache directory. " + 
+                "Matching subdir found in: ${subDirs}")
+            }
+        }
+
     }
 
     docker(options) {
diff --git a/regression-test/suites/cloud_p0/tablets/test_clean_tablet_when_rebalance.groovy b/regression-test/suites/cloud_p0/tablets/test_clean_tablet_when_rebalance.groovy
index 4a44b317cc22338..151de976a8303c2 100644
--- a/regression-test/suites/cloud_p0/tablets/test_clean_tablet_when_rebalance.groovy
+++ b/regression-test/suites/cloud_p0/tablets/test_clean_tablet_when_rebalance.groovy
@@ -32,7 +32,9 @@ suite('test_clean_tablet_when_rebalance', 'docker') {
     ]
     options.feConfigs.add("rehash_tablet_after_be_dead_seconds=$rehashTime")
     options.beConfigs += [
-        'report_tablet_interval_seconds=1'
+        'report_tablet_interval_seconds=1',
+        'write_buffer_size=10240',
+        'write_buffer_size_for_agg=10240'
     ]
     options.setFeNum(3)
     options.setBeNum(3)
@@ -42,56 +44,22 @@ suite('test_clean_tablet_when_rebalance', 'docker') {
     def choseDeadBeIndex = 1
     def table = "test_clean_tablet_when_rebalance"
 
-    def backendIdToHost = { ->
-        def spb = sql_return_maparray """SHOW BACKENDS"""
-        def beIdToHost = [:]
-        spb.each {
-            beIdToHost[it.BackendId] = it.Host
-        }
-        beIdToHost 
-    }
-
-    def getTabletAndBeHostFromFe = { ->
-        def result = sql_return_maparray """SHOW TABLETS FROM $table"""
-        def bes = backendIdToHost.call()
-        // tablet : host
-        def ret = [:]
-        result.each {
-            ret[it.TabletId] = bes[it.BackendId]
-        }
-        ret
-    }
-
-    def getTabletAndBeHostFromBe = { ->
-        def bes = cluster.getAllBackends()
-        def ret = [:]
-        bes.each { be ->
-            // {"msg":"OK","code":0,"data":{"host":"128.2.51.2","tablets":[{"tablet_id":10560},{"tablet_id":10554},{"tablet_id":10552}]},"count":3}
-            def data = Http.GET("http://${be.host}:${be.httpPort}/tablets_json?limit=all", true).data
-            def tablets = data.tablets.collect { it.tablet_id as String }
-            tablets.each{
-                ret[it] = data.host
-            }
-        }
-        ret
-    }
-
-    def testCase = { deadTime -> 
+    def testCase = { deadTime, mergedCacheDir -> 
         boolean beDeadLong = deadTime > rehashTime ? true : false
         logger.info("begin exec beDeadLong {}", beDeadLong)
 
         for (int i = 0; i < 5; i++) {
             sql """
-                select * from $table
+                select count(*) from $table
             """
         }
 
-        def beforeGetFromFe = getTabletAndBeHostFromFe()
-        def beforeGetFromBe = getTabletAndBeHostFromBe.call()
+        def beforeGetFromFe = getTabletAndBeHostFromFe(table)
+        def beforeGetFromBe = getTabletAndBeHostFromBe(cluster.getAllBackends())
         logger.info("before fe tablets {}, be tablets {}", beforeGetFromFe, beforeGetFromBe)
         beforeGetFromFe.each {
             assertTrue(beforeGetFromBe.containsKey(it.Key))
-            assertEquals(beforeGetFromBe[it.Key], it.Value)
+            assertEquals(beforeGetFromBe[it.Key], it.Value[1])
         }
 
         cluster.stopBackends(choseDeadBeIndex)
@@ -120,24 +88,72 @@ suite('test_clean_tablet_when_rebalance', 'docker') {
             bes.size() == (beDeadLong ? 2 : 3)
         }
         for (int i = 0; i < 5; i++) {
+            sleep(2000)
             sql """
-                select * from $table
+                select count(*) from $table
             """
-            sleep(1000)
         }
-        beforeGetFromFe = getTabletAndBeHostFromFe()
-        beforeGetFromBe = getTabletAndBeHostFromBe.call()
-        logger.info("after fe tablets {}, be tablets {}", beforeGetFromFe, beforeGetFromBe)
-        beforeGetFromFe.each {
-            assertTrue(beforeGetFromBe.containsKey(it.Key))
-            assertEquals(beforeGetFromBe[it.Key], it.Value)
+        def afterGetFromFe = getTabletAndBeHostFromFe(table)
+        def afterGetFromBe = getTabletAndBeHostFromBe(cluster.getAllBackends())
+        logger.info("after fe tablets {}, be tablets {}", afterGetFromFe, afterGetFromBe)
+        afterGetFromFe.each {
+            assertTrue(afterGetFromBe.containsKey(it.Key))
+            assertEquals(afterGetFromBe[it.Key], it.Value[1])
+        }
+
+        // TODO(freemandealer)
+        // Once the freemandealer implements file cache cleanup during restart, enabling lines 107 to 145 will allow testing to confirm that after the rebalance, the tablet file cache on the BE will be cleared. In the current implementation, after restarting the BE and triggering the rebalance, the tablets in the tablet manager will be cleared, but the file cache cannot be cleaned up.
+        /*
+        if (beDeadLong) {
+            // check tablet file cache has been deleted
+            // after fe tablets {10309=[10003, 175.41.51.3], 10311=[10002, 175.41.51.2], 10313=[10003, 175.41.51.3]},
+            afterGetFromFe.each {
+                logger.info("tablet_id {}, before host {}, after host {}", it.Key, beforeGetFromFe[it.Key][1], it.Value[1])
+                if (beforeGetFromFe[it.Key][1] == it.Value[1]) {
+                    return
+                }
+                logger.info("tablet_id {} has been reblanced from {} to {}", it.Key, beforeGetFromFe[it.Key][1], it.Value[1])
+                // check before tablet file cache dir has been droped
+
+                def tabletId = it.Key
+                def backendId = beforeGetFromFe[it.Key][0]
+                def backendHost = beforeGetFromFe[it.Key][1]
+                def be = cluster.getBeByBackendId(backendId.toLong())
+                def dataPath = new File("${be.path}/storage/file_cache")
+                def subDirs = []
+                
+                def collectDirs
+                collectDirs = { File dir ->
+                    if (dir.exists()) {
+                        dir.eachDir { subDir ->
+                            subDirs << subDir.name
+                            collectDirs(subDir) 
+                        }
+                    }
+                }
+                
+                collectDirs(dataPath)
+                logger.info("BE {} file_cache subdirs: {}", backendHost, subDirs)
+                def cacheDir = mergedCacheDir[backendHost]
+
+                // add check
+                cacheDir.each { hashFile ->
+                    assertFalse(subDirs.any { subDir -> subDir.startsWith(hashFile) }, 
+                    "Found unexpected cache file pattern ${hashFile} in BE ${backendHost}'s file_cache directory. " + 
+                    "Matching subdir found in: ${subDirs}")
+                }
+            }
         }
+        */
     }
 
     docker(options) {
+        def ms = cluster.getAllMetaservices().get(0)
+        def msHttpPort = ms.host + ":" + ms.httpPort
         sql """CREATE TABLE $table (
             `k1` int(11) NULL,
-            `k2` int(11) NULL
+            `k2` int(11) NULL,
+            `v1` varchar(2048)
             )
             DUPLICATE KEY(`k1`, `k2`)
             COMMENT 'OLAP'
@@ -147,12 +163,13 @@ suite('test_clean_tablet_when_rebalance', 'docker') {
             );
         """
         sql """
-            insert into $table values (1, 1), (2, 2), (3, 3)
+            insert into $table values (1, 1, 'v1'), (2, 2, 'v2'), (3, 3, 'v3')
         """
-        // 'rehash_tablet_after_be_dead_seconds=10'
+        def cacheDirVersion2 = getTabletFileCacheDirFromBe(msHttpPort, table, 2)
+        // 'rehash_tablet_after_be_dead_seconds=100'
         // be-1 dead, but not dead for a long time
-        testCase(5)
+        testCase(5, cacheDirVersion2)
         // be-1 dead, and dead for a long time
-        testCase(200)
+        testCase(200, cacheDirVersion2)
     }
 }

From 617c5e435a7d48fae81ab45820d9c5c395b8fc18 Mon Sep 17 00:00:00 2001
From: airborne12 <jiangkai@selectdb.com>
Date: Mon, 13 Jan 2025 11:40:00 +0800
Subject: [PATCH 135/140] [fix](session var) add need forward for
 disable_inverted_index_v1_for_variant (#46845)

Problem Summary:
disable_inverted_index_v1_for_variant session var need to forward to master.
---
 .../src/main/java/org/apache/doris/qe/SessionVariable.java      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index a52beff44bb73ed..7cdb8ba723682fc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -1257,7 +1257,7 @@ public enum IgnoreSplitType {
     @VariableMgr.VarAttr(name = WAIT_FULL_BLOCK_SCHEDULE_TIMES)
     public int waitFullBlockScheduleTimes = 2;
 
-    @VariableMgr.VarAttr(name = DISABLE_INVERTED_INDEX_V1_FOR_VARIANT)
+    @VariableMgr.VarAttr(name = DISABLE_INVERTED_INDEX_V1_FOR_VARIANT, needForward = true)
     private boolean disableInvertedIndexV1ForVaraint = true;
 
     public int getBeNumberForTest() {

From 1f2aa314095b0526f33e4612b80adba14cfb4612 Mon Sep 17 00:00:00 2001
From: 924060929 <lanhuajian@selectdb.com>
Date: Mon, 13 Jan 2025 11:59:03 +0800
Subject: [PATCH 136/140] [opt](nereids) cache hash code for expression
 (#46763)

cache hash code for expression

this pr can improve planner elapse time from 3.1s to 1.7s when there
have 100k items in `in predicate`
---
 .../org/apache/doris/nereids/analyzer/UnboundFunction.java   | 2 +-
 .../java/org/apache/doris/nereids/analyzer/UnboundSlot.java  | 2 +-
 .../java/org/apache/doris/nereids/analyzer/UnboundStar.java  | 4 ++--
 .../doris/nereids/trees/expressions/AggregateExpression.java | 4 ++--
 .../org/apache/doris/nereids/trees/expressions/Alias.java    | 2 +-
 .../doris/nereids/trees/expressions/ArrayItemReference.java  | 2 +-
 .../org/apache/doris/nereids/trees/expressions/Cast.java     | 4 ++--
 .../org/apache/doris/nereids/trees/expressions/Exists.java   | 2 +-
 .../apache/doris/nereids/trees/expressions/Expression.java   | 5 +++++
 .../apache/doris/nereids/trees/expressions/InPredicate.java  | 2 +-
 .../apache/doris/nereids/trees/expressions/InSubquery.java   | 2 +-
 .../org/apache/doris/nereids/trees/expressions/IsNull.java   | 2 +-
 .../java/org/apache/doris/nereids/trees/expressions/Not.java | 2 +-
 .../doris/nereids/trees/expressions/OrderExpression.java     | 4 ++--
 .../apache/doris/nereids/trees/expressions/Properties.java   | 4 ++--
 .../doris/nereids/trees/expressions/SlotReference.java       | 2 +-
 .../apache/doris/nereids/trees/expressions/SubqueryExpr.java | 2 +-
 .../doris/nereids/trees/expressions/UnaryOperator.java       | 2 +-
 .../org/apache/doris/nereids/trees/expressions/Variable.java | 4 ++--
 .../apache/doris/nereids/trees/expressions/VariableDesc.java | 2 +-
 .../nereids/trees/expressions/VirtualSlotReference.java      | 2 +-
 .../apache/doris/nereids/trees/expressions/WhenClause.java   | 2 +-
 .../doris/nereids/trees/expressions/WindowExpression.java    | 2 +-
 .../apache/doris/nereids/trees/expressions/WindowFrame.java  | 2 +-
 .../nereids/trees/expressions/functions/BoundFunction.java   | 2 +-
 .../trees/expressions/functions/agg/AggregateFunction.java   | 2 +-
 .../expressions/functions/agg/NullableAggregateFunction.java | 4 ++--
 .../trees/expressions/functions/window/WindowFunction.java   | 2 +-
 .../doris/nereids/trees/expressions/literal/Literal.java     | 2 +-
 .../doris/nereids/trees/expressions/literal/NullLiteral.java | 4 ++--
 .../nereids/trees/expressions/literal/StringLikeLiteral.java | 5 -----
 .../nereids/trees/expressions/literal/StructLiteral.java     | 4 ++--
 32 files changed, 44 insertions(+), 44 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundFunction.java
index b4b21e40dcd4df1..9b46293086ad739 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundFunction.java
@@ -162,7 +162,7 @@ public boolean equals(Object o) {
     }
 
     @Override
-    public int hashCode() {
+    public int computeHashCode() {
         return Objects.hash(getName(), isDistinct);
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundSlot.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundSlot.java
index fdcb9547837686c..227aefa1c64412c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundSlot.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundSlot.java
@@ -117,7 +117,7 @@ public boolean equals(Object o) {
     }
 
     @Override
-    public int hashCode() {
+    public int computeHashCode() {
         return nameParts.hashCode();
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java
index cee6a0105f87c7f..67b81b51873d92d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java
@@ -150,8 +150,8 @@ public boolean equals(Object o) {
     }
 
     @Override
-    public int hashCode() {
-        return Objects.hash(super.hashCode(), qualifier, exceptedSlots, replacedAlias);
+    public int computeHashCode() {
+        return Objects.hash(super.computeHashCode(), qualifier, exceptedSlots, replacedAlias);
     }
 
     public UnboundStar withIndexInSql(Pair<Integer, Integer> index) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/AggregateExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/AggregateExpression.java
index 86d7eb723823004..1a2d2765938d325 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/AggregateExpression.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/AggregateExpression.java
@@ -145,8 +145,8 @@ public boolean equals(Object o) {
     }
 
     @Override
-    public int hashCode() {
-        return Objects.hash(super.hashCode(), function, aggregateParam, child());
+    public int computeHashCode() {
+        return Objects.hash(super.computeHashCode(), function, aggregateParam, child());
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java
index 53a82011ac4c3c9..9ff90b5ac96784e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java
@@ -144,7 +144,7 @@ protected boolean extraEquals(Expression other) {
     }
 
     @Override
-    public int hashCode() {
+    public int computeHashCode() {
         return Objects.hash(exprId, qualifier);
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java
index edc074af2b513af..eca81d8202813ab 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java
@@ -121,7 +121,7 @@ public boolean equals(Object o) {
     }
 
     @Override
-    public int hashCode() {
+    public int computeHashCode() {
         return Objects.hash(exprId);
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java
index 0b280af23631923..144e6d829b3a9bd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java
@@ -117,8 +117,8 @@ public boolean equals(Object o) {
     }
 
     @Override
-    public int hashCode() {
-        return Objects.hash(super.hashCode(), targetType);
+    public int computeHashCode() {
+        return Objects.hash(super.computeHashCode(), targetType);
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Exists.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Exists.java
index 8d097d0faa6f45d..dc5a348bec6ed74 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Exists.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Exists.java
@@ -94,7 +94,7 @@ public boolean equals(Object o) {
     }
 
     @Override
-    public int hashCode() {
+    public int computeHashCode() {
         return Objects.hash(this.queryPlan, this.isNot);
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java
index 6634d5e0160ead7..a7b8d41766a9281 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java
@@ -70,6 +70,7 @@ public abstract class Expression extends AbstractTreeNode<Expression> implements
             () -> collect(e -> e instanceof Slot && !(e instanceof ArrayItemSlot)));
     private final int fastChildrenHashCode;
     private final Supplier<String> toSqlCache = Suppliers.memoize(this::computeToSql);
+    private final Supplier<Integer> hashCodeCache = Suppliers.memoize(this::computeHashCode);
 
     protected Expression(Expression... children) {
         super(children);
@@ -440,6 +441,10 @@ protected boolean extraEquals(Expression that) {
 
     @Override
     public int hashCode() {
+        return hashCodeCache.get();
+    }
+
+    protected int computeHashCode() {
         return getClass().hashCode() + fastChildrenHashCode();
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InPredicate.java
index 1f022962787667e..57cc8c73c8b6c92 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InPredicate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InPredicate.java
@@ -144,7 +144,7 @@ public boolean equals(Object o) {
     }
 
     @Override
-    public int hashCode() {
+    protected int computeHashCode() {
         return Objects.hash(compareExpr, options);
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InSubquery.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InSubquery.java
index 71dc1f5eb4f08f0..89b51e374659644 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InSubquery.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InSubquery.java
@@ -127,7 +127,7 @@ public boolean equals(Object o) {
     }
 
     @Override
-    public int hashCode() {
+    public int computeHashCode() {
         return Objects.hash(this.compareExpr, this.listQuery, this.isNot);
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsNull.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsNull.java
index 22216a84bafe874..a5ea5fd9495ff86 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsNull.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsNull.java
@@ -74,7 +74,7 @@ public boolean equals(Object o) {
     }
 
     @Override
-    public int hashCode() {
+    public int computeHashCode() {
         return child().hashCode();
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Not.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Not.java
index b001da9118fea34..25602f08e8ab00c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Not.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Not.java
@@ -92,7 +92,7 @@ public boolean equals(Object o) {
     }
 
     @Override
-    public int hashCode() {
+    public int computeHashCode() {
         return Objects.hash(child().hashCode(), isGeneratedIsNotNull);
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/OrderExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/OrderExpression.java
index 7e33d4315d81adc..b0564b7073761ee 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/OrderExpression.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/OrderExpression.java
@@ -101,7 +101,7 @@ public boolean equals(Object o) {
     }
 
     @Override
-    public int hashCode() {
-        return Objects.hash(super.hashCode(), orderKey);
+    public int computeHashCode() {
+        return Objects.hash(super.computeHashCode(), orderKey);
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Properties.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Properties.java
index d604e919e31d179..09dc6a5c2161789 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Properties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Properties.java
@@ -85,8 +85,8 @@ public boolean equals(Object o) {
     }
 
     @Override
-    public int hashCode() {
-        return Objects.hash(super.hashCode(), keyValues);
+    public int computeHashCode() {
+        return Objects.hash(super.computeHashCode(), keyValues);
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java
index 890fbdfdb96f72c..c742bd35037ccff 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java
@@ -232,7 +232,7 @@ public boolean equals(Object o) {
 
     // The contains method needs to use hashCode, so similar to equals, it only compares exprId
     @Override
-    public int hashCode() {
+    public int computeHashCode() {
         // direct return exprId to speed up
         return exprId.asInt();
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SubqueryExpr.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SubqueryExpr.java
index c08fda1dc6b7135..2011aa9607d1134 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SubqueryExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SubqueryExpr.java
@@ -128,7 +128,7 @@ public boolean equals(Object o) {
     }
 
     @Override
-    public int hashCode() {
+    public int computeHashCode() {
         return Objects.hash(queryPlan, correlateSlots, typeCoercionExpr);
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/UnaryOperator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/UnaryOperator.java
index 61efa91f2621fc2..299e00d0dc4fe4a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/UnaryOperator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/UnaryOperator.java
@@ -56,7 +56,7 @@ public String toString() {
     }
 
     @Override
-    public int hashCode() {
+    public int computeHashCode() {
         return Objects.hash(symbol, child());
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Variable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Variable.java
index 5944ec08744980b..3e753168318caab 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Variable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Variable.java
@@ -112,7 +112,7 @@ public boolean equals(Object o) {
     }
 
     @Override
-    public int hashCode() {
-        return Objects.hash(super.hashCode(), name, type, realExpression);
+    public int computeHashCode() {
+        return Objects.hash(super.computeHashCode(), name, type, realExpression);
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/VariableDesc.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/VariableDesc.java
index 3a16b38f9e19e01..24774afb9889aab 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/VariableDesc.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/VariableDesc.java
@@ -70,7 +70,7 @@ public String toString() {
     }
 
     @Override
-    public int hashCode() {
+    public int computeHashCode() {
         return Objects.hash(isSystemVariable, setType, name);
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/VirtualSlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/VirtualSlotReference.java
index 1b46a8552bafb96..42be621045977b5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/VirtualSlotReference.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/VirtualSlotReference.java
@@ -112,7 +112,7 @@ public boolean equals(Object o) {
     }
 
     @Override
-    public int hashCode() {
+    public int computeHashCode() {
         return Objects.hash(realExpressions, originExpression, getExprId());
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WhenClause.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WhenClause.java
index adb862bb2f10413..6e10701faaad083 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WhenClause.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WhenClause.java
@@ -90,7 +90,7 @@ public List<DataType> expectedInputTypes() {
     }
 
     @Override
-    public int hashCode() {
+    public int computeHashCode() {
         return Objects.hash(left(), right());
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowExpression.java
index 00e7f986ad5826a..d010139d9805e7b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowExpression.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowExpression.java
@@ -174,7 +174,7 @@ public boolean equals(Object o) {
     }
 
     @Override
-    public int hashCode() {
+    public int computeHashCode() {
         return Objects.hash(function, partitionKeys, orderKeys, windowFrame);
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowFrame.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowFrame.java
index 58ed4f15f9baa07..2c0d7d65b80a5fb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowFrame.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowFrame.java
@@ -90,7 +90,7 @@ public boolean equals(Object o) {
     }
 
     @Override
-    public int hashCode() {
+    public int computeHashCode() {
         return Objects.hash(frameUnits, leftBoundary, rightBoundary);
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/BoundFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/BoundFunction.java
index 13d4b515ad75f37..cf936976e82c4d8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/BoundFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/BoundFunction.java
@@ -80,7 +80,7 @@ protected boolean extraEquals(Expression that) {
     }
 
     @Override
-    public int hashCode() {
+    public int computeHashCode() {
         return Objects.hash(getName(), children);
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java
index 777c9c4cc7add5a..7aa5da02ff375e4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java
@@ -92,7 +92,7 @@ public boolean equals(Object o) {
     }
 
     @Override
-    public int hashCode() {
+    public int computeHashCode() {
         return Objects.hash(distinct, getName(), children);
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/NullableAggregateFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/NullableAggregateFunction.java
index fc62710ebb7a61a..82c3599777aa538 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/NullableAggregateFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/NullableAggregateFunction.java
@@ -103,8 +103,8 @@ public boolean equals(Object o) {
     }
 
     @Override
-    public int hashCode() {
-        return Objects.hash(super.hashCode(), alwaysNullable);
+    public int computeHashCode() {
+        return Objects.hash(super.computeHashCode(), alwaysNullable);
     }
 
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/WindowFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/WindowFunction.java
index 57ba66ac5a7e186..1265f685b26e69f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/WindowFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/WindowFunction.java
@@ -52,7 +52,7 @@ public boolean equals(Object o) {
     }
 
     @Override
-    public int hashCode() {
+    public int computeHashCode() {
         return Objects.hash(getName(), children);
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java
index 69e61b03c823140..bb6bbe62a6c2920 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java
@@ -361,7 +361,7 @@ public boolean equals(Object o) {
     }
 
     @Override
-    public int hashCode() {
+    protected int computeHashCode() {
         return Objects.hashCode(getValue());
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/NullLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/NullLiteral.java
index 04da58272881921..5af08e6d130a434 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/NullLiteral.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/NullLiteral.java
@@ -80,7 +80,7 @@ public boolean equals(Object o) {
     }
 
     @Override
-    public int hashCode() {
-        return Objects.hash(super.hashCode(), dataType);
+    protected int computeHashCode() {
+        return Objects.hash(super.computeHashCode(), dataType);
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java
index f5fe33a0ab1bfa6..dba9247fe70213a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java
@@ -74,11 +74,6 @@ public boolean equals(Object o) {
         return Objects.equals(value, that.value);
     }
 
-    @Override
-    public int hashCode() {
-        return Objects.hash(value);
-    }
-
     @Override
     public String toString() {
         return "'" + value + "'";
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StructLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StructLiteral.java
index f44aa663c9eb03c..c9048f288e76bf8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StructLiteral.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StructLiteral.java
@@ -114,8 +114,8 @@ public boolean equals(Object o) {
     }
 
     @Override
-    public int hashCode() {
-        return Objects.hash(super.hashCode(), fields);
+    protected int computeHashCode() {
+        return Objects.hash(super.computeHashCode(), fields);
     }
 
     @Override

From 2fc44514fa86becfcb6bfabcaf437d3dc8e01c0b Mon Sep 17 00:00:00 2001
From: Luwei <luwei@selectdb.com>
Date: Mon, 13 Jan 2025 12:12:38 +0800
Subject: [PATCH 137/140] [enhancement](cloud) get delete bitmap api uses the
 unified rpc retry mechanism (#46848)

---
 be/src/cloud/cloud_meta_mgr.cpp | 45 ++++++++-------------------------
 1 file changed, 11 insertions(+), 34 deletions(-)

diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp
index 2b5f04550cfe2a8..7b3c7f0e933ca91 100644
--- a/be/src/cloud/cloud_meta_mgr.cpp
+++ b/be/src/cloud/cloud_meta_mgr.cpp
@@ -340,6 +340,8 @@ static std::string debug_info(const Request& req) {
     } else if constexpr (is_any_v<Request, RemoveDeleteBitmapUpdateLockRequest>) {
         return fmt::format(" table_id={}, tablet_id={}, lock_id={}", req.table_id(),
                            req.tablet_id(), req.lock_id());
+    } else if constexpr (is_any_v<Request, GetDeleteBitmapRequest>) {
+        return fmt::format(" tablet_id={}", req.tablet_id());
     } else {
         static_assert(!sizeof(Request));
     }
@@ -373,7 +375,11 @@ Status retry_rpc(std::string_view op_name, const Request& req, Response* res,
         std::shared_ptr<MetaService_Stub> stub;
         RETURN_IF_ERROR(proxy->get(&stub));
         brpc::Controller cntl;
-        cntl.set_timeout_ms(config::meta_service_brpc_timeout_ms);
+        if (op_name == "get delete bitmap") {
+            cntl.set_timeout_ms(3 * config::meta_service_brpc_timeout_ms);
+        } else {
+            cntl.set_timeout_ms(config::meta_service_brpc_timeout_ms);
+        }
         cntl.set_max_retry(kBrpcRetryTimes);
         res->Clear();
         (stub.get()->*method)(&cntl, &req, res, nullptr);
@@ -714,41 +720,12 @@ Status CloudMetaMgr::sync_tablet_delete_bitmap(CloudTablet* tablet, int64_t old_
 
     VLOG_DEBUG << "send GetDeleteBitmapRequest: " << req.ShortDebugString();
 
-    int retry_times = 0;
-    MetaServiceProxy* proxy;
-    RETURN_IF_ERROR(MetaServiceProxy::get_proxy(&proxy));
     auto start = std::chrono::high_resolution_clock::now();
-    while (true) {
-        std::shared_ptr<MetaService_Stub> stub;
-        RETURN_IF_ERROR(proxy->get(&stub));
-        // When there are many delete bitmaps that need to be synchronized, it
-        // may take a longer time, especially when loading the tablet for the
-        // first time, so set a relatively long timeout time.
-        brpc::Controller cntl;
-        cntl.set_timeout_ms(3 * config::meta_service_brpc_timeout_ms);
-        cntl.set_max_retry(kBrpcRetryTimes);
-        res.Clear();
-        stub->get_delete_bitmap(&cntl, &req, &res, nullptr);
-        if (cntl.Failed()) [[unlikely]] {
-            LOG_INFO("failed to get delete bitmap")
-                    .tag("reason", cntl.ErrorText())
-                    .tag("tablet_id", tablet->tablet_id())
-                    .tag("partition_id", tablet->partition_id())
-                    .tag("tried", retry_times);
-            proxy->set_unhealthy();
-        } else {
-            break;
-        }
-
-        if (++retry_times > config::delete_bitmap_rpc_retry_times) {
-            if (cntl.Failed()) {
-                return Status::RpcError("failed to get delete bitmap, tablet={} err={}",
-                                        tablet->tablet_id(), cntl.ErrorText());
-            }
-            break;
-        }
-    }
+    auto st = retry_rpc("get delete bitmap", req, &res, &MetaService_Stub::get_delete_bitmap);
     auto end = std::chrono::high_resolution_clock::now();
+    if (st.code() == ErrorCode::THRIFT_RPC_ERROR) {
+        return st;
+    }
 
     if (res.status().code() == MetaServiceCode::TABLET_NOT_FOUND) {
         return Status::NotFound("failed to get delete bitmap: {}", res.status().msg());

From 1cae7eb5f68ba76ebfa3fb6760fa2d5c94540a0f Mon Sep 17 00:00:00 2001
From: shuke <shuke@selectdb.com>
Date: Mon, 13 Jan 2025 12:31:18 +0800
Subject: [PATCH 138/140] =?UTF-8?q?[regression-test](framework=EF=BC=89fix?=
 =?UTF-8?q?=20master=5Fsql=20action=20bug=20in=20framwork=20(#46874)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../groovy/org/apache/doris/regression/suite/Suite.groovy     | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index 2b121cb8f4497d4..52682f740b89887 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -1326,6 +1326,8 @@ class Suite implements GroovyInterceptable {
                     tupleResult = JdbcUtils.executeToStringList(context.getArrowFlightSqlConnection(), (PreparedStatement) arg)
                 } else if (tag.contains("target_sql")) {
                     tupleResult = JdbcUtils.executeToStringList(context.getTargetConnection(this), (PreparedStatement) arg)
+                } else if (tag.contains("master_sql")) {
+                    tupleResult = JdbcUtils.executeToStringList(context.getMasterConnection(), (PreparedStatement) arg)
                 } else {
                     tupleResult = JdbcUtils.executeToStringList(context.getConnection(),  (PreparedStatement) arg)
                 }
@@ -1339,6 +1341,8 @@ class Suite implements GroovyInterceptable {
                             (String) ("USE ${context.dbName};" + (String) arg))
                 } else if (tag.contains("target_sql")) {
                     tupleResult = JdbcUtils.executeToStringList(context.getTargetConnection(this), (String) arg)
+                } else if (tag.contains("master_sql")) {
+                    tupleResult = JdbcUtils.executeToStringList(context.getMasterConnection(), (String) arg)
                 } else {
                     tupleResult = JdbcUtils.executeToStringList(context.getConnection(),  (String) arg)
                 }

From 0f95ab3894e9a09946b8c7061089bd675b8bc79c Mon Sep 17 00:00:00 2001
From: zhangstar333 <zhangsida@selectdb.com>
Date: Mon, 13 Jan 2025 14:25:00 +0800
Subject: [PATCH 139/140] [refactor](partition) refactor partition sort node
 about control data to output  (#46300)

### What problem does this PR solve?
Problem Summary:

before when the partition sort sink operator all of sorters have been
sorted data
then set source operator decency ready.
but the sorted data maybe need more time, so we could notice source
operator after the first sorter have done.
---
 .../exec/partition_sort_sink_operator.cpp     | 20 +++++++++++----
 .../exec/partition_sort_sink_operator.h       |  1 +
 .../exec/partition_sort_source_operator.cpp   | 25 ++++++++++---------
 be/src/vec/common/sort/partition_sorter.cpp   |  2 ++
 be/src/vec/common/sort/partition_sorter.h     |  3 +++
 5 files changed, 34 insertions(+), 17 deletions(-)

diff --git a/be/src/pipeline/exec/partition_sort_sink_operator.cpp b/be/src/pipeline/exec/partition_sort_sink_operator.cpp
index cdc4b31c76d7e06..2b5916c3e585493 100644
--- a/be/src/pipeline/exec/partition_sort_sink_operator.cpp
+++ b/be/src/pipeline/exec/partition_sort_sink_operator.cpp
@@ -17,6 +17,8 @@
 
 #include "partition_sort_sink_operator.h"
 
+#include <glog/logging.h>
+
 #include <cstdint>
 
 #include "common/status.h"
@@ -47,6 +49,7 @@ Status PartitionSortSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo
     _build_timer = ADD_TIMER(_profile, "HashTableBuildTime");
     _selector_block_timer = ADD_TIMER(_profile, "SelectorBlockTime");
     _emplace_key_timer = ADD_TIMER(_profile, "EmplaceKeyTime");
+    _sorted_data_timer = ADD_TIMER(_profile, "SortedDataTime");
     _passthrough_rows_counter = ADD_COUNTER(_profile, "PassThroughRowsCounter", TUnit::UNIT);
     _sorted_partition_input_rows_counter =
             ADD_COUNTER(_profile, "SortedPartitionInputRows", TUnit::UNIT);
@@ -134,17 +137,23 @@ Status PartitionSortSinkOperatorX::sink(RuntimeState* state, vectorized::Block*
         //seems could free for hashtable
         local_state._agg_arena_pool.reset(nullptr);
         local_state._partitioned_data.reset(nullptr);
+        SCOPED_TIMER(local_state._sorted_data_timer);
+        for (auto& _value_place : local_state._value_places) {
+            _value_place->create_or_reset_sorter_state();
+            local_state._shared_state->partition_sorts.emplace_back(
+                    std::move(_value_place->_partition_topn_sorter));
+        }
+        // notice: need split two for loop, as maybe need check sorter early
         for (int i = 0; i < local_state._value_places.size(); ++i) {
-            local_state._value_places[i]->create_or_reset_sorter_state();
-            auto sorter = std::move(local_state._value_places[i]->_partition_topn_sorter);
-
-            //get blocks from every partition, and sorter get those data.
+            auto& sorter = local_state._shared_state->partition_sorts[i];
             for (const auto& block : local_state._value_places[i]->_blocks) {
                 RETURN_IF_ERROR(sorter->append_block(block.get()));
             }
             local_state._value_places[i]->_blocks.clear();
             RETURN_IF_ERROR(sorter->prepare_for_read());
-            local_state._shared_state->partition_sorts.push_back(std::move(sorter));
+            // iff one sorter have data, then could set source ready to read
+            std::unique_lock<std::mutex> lc(local_state._shared_state->sink_eos_lock);
+            local_state._dependency->set_ready_to_read();
         }
 
         COUNTER_SET(local_state._hash_table_size_counter, int64_t(local_state._num_partition));
@@ -154,6 +163,7 @@ Status PartitionSortSinkOperatorX::sink(RuntimeState* state, vectorized::Block*
         {
             std::unique_lock<std::mutex> lc(local_state._shared_state->sink_eos_lock);
             local_state._shared_state->sink_eos = true;
+            // this ready is also need, as source maybe block by self in some case
             local_state._dependency->set_ready_to_read();
         }
         local_state._profile->add_info_string("HasPassThrough",
diff --git a/be/src/pipeline/exec/partition_sort_sink_operator.h b/be/src/pipeline/exec/partition_sort_sink_operator.h
index 32bbf38202713f1..3196afeb149b271 100644
--- a/be/src/pipeline/exec/partition_sort_sink_operator.h
+++ b/be/src/pipeline/exec/partition_sort_sink_operator.h
@@ -58,6 +58,7 @@ class PartitionSortSinkLocalState : public PipelineXSinkLocalState<PartitionSort
     RuntimeProfile::Counter* _build_timer = nullptr;
     RuntimeProfile::Counter* _emplace_key_timer = nullptr;
     RuntimeProfile::Counter* _selector_block_timer = nullptr;
+    RuntimeProfile::Counter* _sorted_data_timer = nullptr;
     RuntimeProfile::Counter* _hash_table_size_counter = nullptr;
     RuntimeProfile::Counter* _passthrough_rows_counter = nullptr;
     RuntimeProfile::Counter* _sorted_partition_input_rows_counter = nullptr;
diff --git a/be/src/pipeline/exec/partition_sort_source_operator.cpp b/be/src/pipeline/exec/partition_sort_source_operator.cpp
index e4f798c433ffa86..41ba4811034d57f 100644
--- a/be/src/pipeline/exec/partition_sort_source_operator.cpp
+++ b/be/src/pipeline/exec/partition_sort_source_operator.cpp
@@ -41,7 +41,6 @@ Status PartitionSortSourceOperatorX::get_block(RuntimeState* state, vectorized::
     auto& local_state = get_local_state(state);
     SCOPED_TIMER(local_state.exec_time_counter());
     output_block->clear_column_data();
-
     auto get_data_from_blocks_buffer = false;
     {
         std::lock_guard<std::mutex> lock(local_state._shared_state->buffer_mutex);
@@ -55,6 +54,7 @@ Status PartitionSortSourceOperatorX::get_block(RuntimeState* state, vectorized::
                 // add this mutex to check, as in some case maybe is doing block(), and the sink is doing set eos.
                 // so have to hold mutex to set block(), avoid to sink have set eos and set ready, but here set block() by mistake
                 std::unique_lock<std::mutex> lc(local_state._shared_state->sink_eos_lock);
+                //if buffer have no data and sink not eos, block reading and wait for signal again
                 if (!local_state._shared_state->sink_eos) {
                     local_state._dependency->block();
                 }
@@ -69,16 +69,10 @@ Status PartitionSortSourceOperatorX::get_block(RuntimeState* state, vectorized::
         // if we move the _blocks_buffer output at last(behind 286 line),
         // it's maybe eos but not output all data: when _blocks_buffer.empty() and _can_read = false (this: _sort_idx && _partition_sorts.size() are 0)
         RETURN_IF_ERROR(get_sorted_block(state, output_block, local_state));
-        {
-            std::lock_guard<std::mutex> lock(local_state._shared_state->buffer_mutex);
-
-            *eos = local_state._shared_state->blocks_buffer.empty() &&
-                   local_state._sort_idx >= local_state._shared_state->partition_sorts.size();
-        }
+        *eos = local_state._sort_idx >= local_state._shared_state->partition_sorts.size();
     }
 
     if (!output_block->empty()) {
-        //if buffer have no data and sink not eos, block reading and wait for signal again
         RETURN_IF_ERROR(vectorized::VExprContext::filter_block(local_state._conjuncts, output_block,
                                                                output_block->columns()));
         local_state._num_rows_returned += output_block->rows();
@@ -92,15 +86,22 @@ Status PartitionSortSourceOperatorX::get_sorted_block(RuntimeState* state,
     SCOPED_TIMER(local_state._get_sorted_timer);
     //sorter output data one by one
     bool current_eos = false;
-    if (local_state._sort_idx < local_state._shared_state->partition_sorts.size()) {
-        RETURN_IF_ERROR(local_state._shared_state->partition_sorts[local_state._sort_idx]->get_next(
-                state, output_block, &current_eos));
+    auto& sorters = local_state._shared_state->partition_sorts;
+    auto sorter_size = sorters.size();
+    if (local_state._sort_idx < sorter_size) {
+        RETURN_IF_ERROR(
+                sorters[local_state._sort_idx]->get_next(state, output_block, &current_eos));
         COUNTER_UPDATE(local_state._sorted_partition_output_rows_counter, output_block->rows());
     }
     if (current_eos) {
         // current sort have eos, so get next idx
-        local_state._shared_state->partition_sorts[local_state._sort_idx].reset(nullptr);
+        sorters[local_state._sort_idx].reset(nullptr);
         local_state._sort_idx++;
+        std::unique_lock<std::mutex> lc(local_state._shared_state->sink_eos_lock);
+        if (local_state._sort_idx < sorter_size &&
+            !sorters[local_state._sort_idx]->prepared_finish()) {
+            local_state._dependency->block();
+        }
     }
 
     return Status::OK();
diff --git a/be/src/vec/common/sort/partition_sorter.cpp b/be/src/vec/common/sort/partition_sorter.cpp
index f6a3cb443aae77e..c034c8a72d17088 100644
--- a/be/src/vec/common/sort/partition_sorter.cpp
+++ b/be/src/vec/common/sort/partition_sorter.cpp
@@ -74,6 +74,7 @@ Status PartitionSorter::prepare_for_read() {
     }
     queue = MergeSorterQueue(cursors);
     blocks.clear();
+    _prepared_finish = true;
     return Status::OK();
 }
 
@@ -90,6 +91,7 @@ void PartitionSorter::reset_sorter_state(RuntimeState* runtime_state) {
     }
     _output_total_rows = 0;
     _output_distinct_rows = 0;
+    _prepared_finish = false;
 }
 
 Status PartitionSorter::get_next(RuntimeState* state, Block* block, bool* eos) {
diff --git a/be/src/vec/common/sort/partition_sorter.h b/be/src/vec/common/sort/partition_sorter.h
index 053b3aa1a29bbdc..6cfc8d7be93ad13 100644
--- a/be/src/vec/common/sort/partition_sorter.h
+++ b/be/src/vec/common/sort/partition_sorter.h
@@ -20,6 +20,7 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#include <atomic>
 #include <cstdint>
 #include <memory>
 #include <vector>
@@ -92,6 +93,7 @@ class PartitionSorter final : public Sorter {
     size_t data_size() const override { return _state->data_size(); }
     int64 get_output_rows() const { return _output_total_rows; }
     void reset_sorter_state(RuntimeState* runtime_state);
+    bool prepared_finish() { return _prepared_finish; }
 
 private:
     Status _read_row_num(Block* block, bool* eos, int batch_size);
@@ -117,6 +119,7 @@ class PartitionSorter final : public Sorter {
     int _partition_inner_limit = 0;
     TopNAlgorithm::type _top_n_algorithm = TopNAlgorithm::type::ROW_NUMBER;
     SortCursorCmp* _previous_row = nullptr;
+    std::atomic_bool _prepared_finish = false;
 };
 
 } // namespace doris::vectorized

From ca6b229ba5c40c8c40429b5ef036f83f5817cd40 Mon Sep 17 00:00:00 2001
From: bobhan1 <baohan@selectdb.com>
Date: Mon, 13 Jan 2025 14:27:50 +0800
Subject: [PATCH 140/140] [Fix](cloud) Remove pending delete bitmap's `lock_id`
 check when commit txn in MS (#46841)

### What problem does this PR solve?

Related PR: https://github.com/apache/doris/pull/46039

Problem Summary:

https://github.com/apache/doris/pull/46039 add a defensive check when
commit_txn in MS to check whether the `lock_id` of pending delete
bitmaps on tablets involved in the txn is the current txn's `lock_id`.
But this may report a false negative in the following circumstance:

1. heavy schema change begins and add shadow index to table.
2. txn A load data to base index and shadow index.
3. txn A write its pending delete bitmaps on MS. This includes tablets
of base index and shadow index.
4. txn A failed to remove its pending delete bitmaps for some reson(e.g.
`commit_txn()` failed due to too large value)
5. txn B load data to base index and shadow index.
6. schema change failed for some reason and **remove shadow index on
table.**
7. txn B send delete bitmap calculation task to BE. **Note that this
will not involved tablets under shadow index because these tablets have
been dropped.** **So these tablets' pending delete bitmaps will still be
txn A's**.
8. txn B commit txn on MS and find that pending delete bitmaps'
`lock_id` on tablets under shadow index not match. And txn B will
failed.

We can see that the checks on these dropped tablets are useless so we
remove the mandatory check to avoid this false negative and print a
warning log instead to help locate problems.
---
 cloud/src/meta-service/meta_service_txn.cpp |  6 +++---
 cloud/test/meta_service_test.cpp            | 17 ++++++++++++++++-
 2 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp
index 34657cbdb821ce3..e5c22ff401b7eb0 100644
--- a/cloud/src/meta-service/meta_service_txn.cpp
+++ b/cloud/src/meta-service/meta_service_txn.cpp
@@ -997,12 +997,12 @@ void process_mow_when_commit_txn(
             }
 
             if (pending_info.has_lock_id() && pending_info.lock_id() != lock_id) {
-                code = MetaServiceCode::PENDING_DELETE_BITMAP_WRONG;
-                msg = fmt::format(
+                TEST_SYNC_POINT_CALLBACK("commit_txn:check_pending_delete_bitmap_lock_id",
+                                         &tablet_id);
+                LOG_WARNING(
                         "wrong lock_id in pending delete bitmap infos, expect lock_id={}, but "
                         "found {} tablet_id={} instance_id={}",
                         lock_id, pending_info.lock_id(), tablet_id, instance_id);
-                return;
             }
 
             txn->remove(pending_key);
diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp
index 49b1587228c9120..b7004716035a465 100644
--- a/cloud/test/meta_service_test.cpp
+++ b/cloud/test/meta_service_test.cpp
@@ -5479,6 +5479,15 @@ TEST(MetaServiceTest, WrongPendingBitmapTest) {
                                        const std::string& cloud_unique_id);
     auto instance_id = get_instance_id(meta_service->resource_mgr(), "test_cloud_unique_id");
 
+    std::set<int64_t> real_wrong_pending_delete_bitmap_tablet_ids;
+    std::set<int64_t> expected_wrong_pending_delete_bitmap_tablet_ids;
+    auto sp = SyncPoint::get_instance();
+    sp->set_call_back("commit_txn:check_pending_delete_bitmap_lock_id", [&](auto&& args) {
+        auto* tablet_id = try_any_cast<int64_t*>(args[0]);
+        real_wrong_pending_delete_bitmap_tablet_ids.insert(*tablet_id);
+    });
+    sp->enable_processing();
+
     // case: first version of rowset
     {
         int64_t txn_id = 56789;
@@ -5587,6 +5596,8 @@ TEST(MetaServiceTest, WrongPendingBitmapTest) {
             ASSERT_TRUE(pending_info.SerializeToString(&pending_val));
             txn->put(pending_key, pending_val);
             ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+
+            expected_wrong_pending_delete_bitmap_tablet_ids.insert(tablet_id_base);
         }
 
         // commit txn
@@ -5600,9 +5611,13 @@ TEST(MetaServiceTest, WrongPendingBitmapTest) {
             CommitTxnResponse res;
             meta_service->commit_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl),
                                      &req, &res, nullptr);
-            ASSERT_EQ(res.status().code(), MetaServiceCode::PENDING_DELETE_BITMAP_WRONG);
+            ASSERT_EQ(expected_wrong_pending_delete_bitmap_tablet_ids,
+                      real_wrong_pending_delete_bitmap_tablet_ids);
         }
     }
+
+    SyncPoint::get_instance()->disable_processing();
+    SyncPoint::get_instance()->clear_all_call_backs();
 }
 
 TEST(MetaServiceTest, GetDeleteBitmapWithRetryTest1) {