diff --git a/.asf.yaml b/.asf.yaml index 7a7d845e4c9bb0..e75df33d1919a6 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -88,9 +88,10 @@ github: - P0 Regression (Doris Regression) - External Regression (Doris External Regression) - cloud_p0 (Doris Cloud Regression) - #required_pull_request_reviews: - # dismiss_stale_reviews: true - # required_approving_review_count: 1 + required_pull_request_reviews: + require_code_owner_reviews: true + required_approving_review_count: 1 + dismiss_stale_reviews: true branch-2.1: required_status_checks: diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b00919e4c8728d..bed8224817dd5e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -14,8 +14,24 @@ # See the License for the specific language governing permissions and # limitations under the License. # -be/src/io/* @platoneko @gavinchou @dataroaring + +# for more syntax help and usage example +# https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners#codeowners-syntax + +be/src/io/ @gavinchou @dataroaring be/src/agent/be_exec_version_manager.cpp @BiteTheDDDDt fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @dataroaring @CalvinKirs @morningman **/pom.xml @CalvinKirs @morningman fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java @dataroaring @morningman @yiguolei @xiaokang +fe/fe-core/src/main/java/org/apache/doris/cloud/ @luwei16 @gavinchou @dearding @dataroaring +cloud/src/meta-service/*job* @luwei16 @gavinchou @dataroaring +cloud/src/meta-service/*meta_service_txn* @swjtu-zhanglei @gavinchou @dataroaring +cloud/src/meta-service/*resource* @deardeng @gavinchou @dataroaring +cloud/src/resource-manager/* @deardeng @gavinchou @dataroaring +cloud/src/recycler/* @swjtu-zhanglei @gavinchou @dataroaring +be/src/cloud/*compact* @luwei16 @gavinchou @dataroaring +be/src/cloud/*schema* @luwei16 @gavinchou @dataroaring +cloud/ @gavinchou @dataroaring @w41ter +be/src/cloud/ @gavinchou @dataroaring +gensrc/proto/olap_file.proto @gavinchou @dataroaring @yiguolei +gensrc/proto/cloud.proto @gavinchou @dataroaring @w41ter diff --git a/be/src/cloud/cloud_schema_change_job.cpp b/be/src/cloud/cloud_schema_change_job.cpp index 81b07ff7e4c843..d12bcdaa01e21c 100644 --- a/be/src/cloud/cloud_schema_change_job.cpp +++ b/be/src/cloud/cloud_schema_change_job.cpp @@ -395,7 +395,7 @@ Status CloudSchemaChangeJob::_convert_historical_rowsets(const SchemaChangeParam DBUG_EXECUTE_IF("CloudSchemaChangeJob::_convert_historical_rowsets.test_conflict", { std::srand(static_cast(std::time(nullptr))); int random_value = std::rand() % 100; - if (random_value < 50) { + if (random_value < 20) { return Status::Error("test txn conflict"); } }); diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp index b85f370165a3de..58ec8cce5c4993 100644 --- a/be/src/exprs/runtime_filter.cpp +++ b/be/src/exprs/runtime_filter.cpp @@ -283,711 +283,6 @@ Status create_vbin_predicate(const TypeDescriptor& type, TExprOpcode::type opcod *tnode = node; return vectorized::VExpr::create_expr(node, expr); } -// This class is a wrapper of runtime predicate function -class RuntimePredicateWrapper { -public: - RuntimePredicateWrapper(const RuntimeFilterParams* params) - : RuntimePredicateWrapper(params->column_return_type, params->filter_type, - params->filter_id) {}; - // for a 'tmp' runtime predicate wrapper - // only could called assign method or as a param for merge - RuntimePredicateWrapper(PrimitiveType column_type, RuntimeFilterType type, uint32_t filter_id) - : _column_return_type(column_type), - _filter_type(type), - _context(new RuntimeFilterContext()), - _filter_id(filter_id) {} - - // init runtime filter wrapper - // alloc memory to init runtime filter function - Status init(const RuntimeFilterParams* params) { - _max_in_num = params->max_in_num; - switch (_filter_type) { - case RuntimeFilterType::IN_FILTER: { - _context->hybrid_set.reset(create_set(_column_return_type)); - _context->hybrid_set->set_null_aware(params->null_aware); - break; - } - // Only use in nested loop join not need set null aware - case RuntimeFilterType::MIN_FILTER: - case RuntimeFilterType::MAX_FILTER: { - _context->minmax_func.reset(create_minmax_filter(_column_return_type)); - break; - } - case RuntimeFilterType::MINMAX_FILTER: { - _context->minmax_func.reset(create_minmax_filter(_column_return_type)); - _context->minmax_func->set_null_aware(params->null_aware); - break; - } - case RuntimeFilterType::BLOOM_FILTER: { - _context->bloom_filter_func.reset(create_bloom_filter(_column_return_type)); - _context->bloom_filter_func->init_params(params); - return Status::OK(); - } - case RuntimeFilterType::IN_OR_BLOOM_FILTER: { - _context->hybrid_set.reset(create_set(_column_return_type)); - _context->hybrid_set->set_null_aware(params->null_aware); - _context->bloom_filter_func.reset(create_bloom_filter(_column_return_type)); - _context->bloom_filter_func->init_params(params); - return Status::OK(); - } - case RuntimeFilterType::BITMAP_FILTER: { - _context->bitmap_filter_func.reset(create_bitmap_filter(_column_return_type)); - _context->bitmap_filter_func->set_not_in(params->bitmap_filter_not_in); - return Status::OK(); - } - default: - return Status::InternalError("Unknown Filter type"); - } - return Status::OK(); - } - - Status change_to_bloom_filter() { - if (_filter_type != RuntimeFilterType::IN_OR_BLOOM_FILTER) { - return Status::InternalError( - "Can not change to bloom filter because of runtime filter type is {}", - IRuntimeFilter::to_string(_filter_type)); - } - BloomFilterFuncBase* bf = _context->bloom_filter_func.get(); - - if (bf != nullptr) { - insert_to_bloom_filter(bf); - } else if (_context->hybrid_set != nullptr && _context->hybrid_set->size() != 0) { - return Status::InternalError("change to bloom filter need empty set ", - IRuntimeFilter::to_string(_filter_type)); - } - - // release in filter - _context->hybrid_set.reset(); - return Status::OK(); - } - - Status init_bloom_filter(const size_t build_bf_cardinality) { - if (_filter_type != RuntimeFilterType::BLOOM_FILTER && - _filter_type != RuntimeFilterType::IN_OR_BLOOM_FILTER) { - throw Exception(ErrorCode::INTERNAL_ERROR, - "init_bloom_filter meet invalid input type {}", int(_filter_type)); - } - return _context->bloom_filter_func->init_with_cardinality(build_bf_cardinality); - } - - bool get_build_bf_cardinality() const { - if (_filter_type == RuntimeFilterType::BLOOM_FILTER || - _filter_type == RuntimeFilterType::IN_OR_BLOOM_FILTER) { - return _context->bloom_filter_func->get_build_bf_cardinality(); - } - return false; - } - - void insert_to_bloom_filter(BloomFilterFuncBase* bloom_filter) const { - if (_context->hybrid_set->size() > 0) { - auto* it = _context->hybrid_set->begin(); - while (it->has_next()) { - bloom_filter->insert(it->get_value()); - it->next(); - } - } - if (_context->hybrid_set->contain_null()) { - bloom_filter->set_contain_null_and_null_aware(); - } - } - - BloomFilterFuncBase* get_bloomfilter() const { return _context->bloom_filter_func.get(); } - - void insert_fixed_len(const vectorized::ColumnPtr& column, size_t start) { - if (is_ignored()) { - throw Exception(ErrorCode::INTERNAL_ERROR, "insert_fixed_len meet ignored rf"); - } - switch (_filter_type) { - case RuntimeFilterType::IN_FILTER: { - _context->hybrid_set->insert_fixed_len(column, start); - break; - } - case RuntimeFilterType::MIN_FILTER: - case RuntimeFilterType::MAX_FILTER: - case RuntimeFilterType::MINMAX_FILTER: { - _context->minmax_func->insert_fixed_len(column, start); - break; - } - case RuntimeFilterType::BLOOM_FILTER: { - _context->bloom_filter_func->insert_fixed_len(column, start); - break; - } - case RuntimeFilterType::IN_OR_BLOOM_FILTER: { - if (is_bloomfilter()) { - _context->bloom_filter_func->insert_fixed_len(column, start); - } else { - _context->hybrid_set->insert_fixed_len(column, start); - } - break; - } - default: - DCHECK(false); - break; - } - } - - void insert_batch(const vectorized::ColumnPtr& column, size_t start) { - if (get_real_type() == RuntimeFilterType::BITMAP_FILTER) { - bitmap_filter_insert_batch(column, start); - } else { - insert_fixed_len(column, start); - } - } - - void bitmap_filter_insert_batch(const vectorized::ColumnPtr column, size_t start) { - std::vector bitmaps; - if (column->is_nullable()) { - const auto* nullable = assert_cast(column.get()); - const auto& col = - assert_cast(nullable->get_nested_column()); - const auto& nullmap = - assert_cast(nullable->get_null_map_column()) - .get_data(); - for (size_t i = start; i < column->size(); i++) { - if (!nullmap[i]) { - bitmaps.push_back(&(col.get_data()[i])); - } - } - } else { - const auto* col = assert_cast(column.get()); - for (size_t i = start; i < column->size(); i++) { - bitmaps.push_back(&(col->get_data()[i])); - } - } - _context->bitmap_filter_func->insert_many(bitmaps); - } - - RuntimeFilterType get_real_type() const { - if (_filter_type == RuntimeFilterType::IN_OR_BLOOM_FILTER) { - if (_context->hybrid_set) { - return RuntimeFilterType::IN_FILTER; - } - return RuntimeFilterType::BLOOM_FILTER; - } - return _filter_type; - } - - size_t get_bloom_filter_size() const { - return _context->bloom_filter_func ? _context->bloom_filter_func->get_size() : 0; - } - - Status get_push_exprs(std::list& probe_ctxs, - std::vector& push_exprs, - const TExpr& probe_expr); - - Status merge(const RuntimePredicateWrapper* wrapper) { - if (wrapper->is_disabled()) { - set_disabled(); - return Status::OK(); - } - - if (wrapper->is_ignored() || is_disabled()) { - return Status::OK(); - } - - _context->ignored = false; - - bool can_not_merge_in_or_bloom = - _filter_type == RuntimeFilterType::IN_OR_BLOOM_FILTER && - (wrapper->_filter_type != RuntimeFilterType::IN_FILTER && - wrapper->_filter_type != RuntimeFilterType::BLOOM_FILTER && - wrapper->_filter_type != RuntimeFilterType::IN_OR_BLOOM_FILTER); - - bool can_not_merge_other = _filter_type != RuntimeFilterType::IN_OR_BLOOM_FILTER && - _filter_type != wrapper->_filter_type; - - CHECK(!can_not_merge_in_or_bloom && !can_not_merge_other) - << " can not merge runtime filter(id=" << _filter_id - << "), current is filter type is " << IRuntimeFilter::to_string(_filter_type) - << ", other filter type is " << IRuntimeFilter::to_string(wrapper->_filter_type); - - switch (_filter_type) { - case RuntimeFilterType::IN_FILTER: { - _context->hybrid_set->insert(wrapper->_context->hybrid_set.get()); - if (_max_in_num >= 0 && _context->hybrid_set->size() >= _max_in_num) { - set_disabled(); - } - break; - } - case RuntimeFilterType::MIN_FILTER: - case RuntimeFilterType::MAX_FILTER: - case RuntimeFilterType::MINMAX_FILTER: { - RETURN_IF_ERROR(_context->minmax_func->merge(wrapper->_context->minmax_func.get())); - break; - } - case RuntimeFilterType::BLOOM_FILTER: { - RETURN_IF_ERROR( - _context->bloom_filter_func->merge(wrapper->_context->bloom_filter_func.get())); - break; - } - case RuntimeFilterType::IN_OR_BLOOM_FILTER: { - auto real_filter_type = get_real_type(); - - auto other_filter_type = wrapper->_filter_type; - if (other_filter_type == RuntimeFilterType::IN_OR_BLOOM_FILTER) { - other_filter_type = wrapper->get_real_type(); - } - - if (real_filter_type == RuntimeFilterType::IN_FILTER) { - // when we meet base rf is in-filter, threre only have two case: - // case1: all input-filter's build_bf_exactly is true, inited by synced global size - // case2: all input-filter's build_bf_exactly is false, inited by default size - if (other_filter_type == RuntimeFilterType::IN_FILTER) { - _context->hybrid_set->insert(wrapper->_context->hybrid_set.get()); - if (_max_in_num >= 0 && _context->hybrid_set->size() >= _max_in_num) { - // case2: use default size to init bf - RETURN_IF_ERROR(_context->bloom_filter_func->init_with_fixed_length()); - RETURN_IF_ERROR(change_to_bloom_filter()); - } - } else { - // case1&case2: use input bf directly and insert hybrid set data into bf - _context->bloom_filter_func = wrapper->_context->bloom_filter_func; - RETURN_IF_ERROR(change_to_bloom_filter()); - } - } else { - if (other_filter_type == RuntimeFilterType::IN_FILTER) { - // case2: insert data to global filter - wrapper->insert_to_bloom_filter(_context->bloom_filter_func.get()); - } else { - // case1&case2: all input bf must has same size - RETURN_IF_ERROR(_context->bloom_filter_func->merge( - wrapper->_context->bloom_filter_func.get())); - } - } - break; - } - case RuntimeFilterType::BITMAP_FILTER: { - // use input bitmap directly because we assume bitmap filter join always have full data - _context->bitmap_filter_func = wrapper->_context->bitmap_filter_func; - break; - } - default: - return Status::InternalError("unknown runtime filter"); - } - return Status::OK(); - } - - Status assign(const PInFilter* in_filter, bool contain_null) { - _context->hybrid_set.reset(create_set(_column_return_type)); - if (contain_null) { - _context->hybrid_set->set_null_aware(true); - _context->hybrid_set->insert((const void*)nullptr); - } - - switch (_column_return_type) { - case TYPE_BOOLEAN: { - batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { - bool bool_val = column.boolval(); - set->insert(&bool_val); - }); - break; - } - case TYPE_TINYINT: { - batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { - auto int_val = static_cast(column.intval()); - set->insert(&int_val); - }); - break; - } - case TYPE_SMALLINT: { - batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { - auto int_val = static_cast(column.intval()); - set->insert(&int_val); - }); - break; - } - case TYPE_INT: { - batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { - int32_t int_val = column.intval(); - set->insert(&int_val); - }); - break; - } - case TYPE_BIGINT: { - batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { - int64_t long_val = column.longval(); - set->insert(&long_val); - }); - break; - } - case TYPE_LARGEINT: { - batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { - auto string_val = column.stringval(); - StringParser::ParseResult result; - auto int128_val = StringParser::string_to_int( - string_val.c_str(), string_val.length(), &result); - DCHECK(result == StringParser::PARSE_SUCCESS); - set->insert(&int128_val); - }); - break; - } - case TYPE_FLOAT: { - batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { - auto float_val = static_cast(column.doubleval()); - set->insert(&float_val); - }); - break; - } - case TYPE_DOUBLE: { - batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { - double double_val = column.doubleval(); - set->insert(&double_val); - }); - break; - } - case TYPE_DATEV2: { - batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { - auto date_v2_val = column.intval(); - set->insert(&date_v2_val); - }); - break; - } - case TYPE_DATETIMEV2: { - batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { - auto date_v2_val = column.longval(); - set->insert(&date_v2_val); - }); - break; - } - case TYPE_DATETIME: - case TYPE_DATE: { - batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { - const auto& string_val_ref = column.stringval(); - VecDateTimeValue datetime_val; - datetime_val.from_date_str(string_val_ref.c_str(), string_val_ref.length()); - set->insert(&datetime_val); - }); - break; - } - case TYPE_DECIMALV2: { - batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { - const auto& string_val_ref = column.stringval(); - DecimalV2Value decimal_val(string_val_ref); - set->insert(&decimal_val); - }); - break; - } - case TYPE_DECIMAL32: { - batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { - int32_t decimal_32_val = column.intval(); - set->insert(&decimal_32_val); - }); - break; - } - case TYPE_DECIMAL64: { - batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { - int64_t decimal_64_val = column.longval(); - set->insert(&decimal_64_val); - }); - break; - } - case TYPE_DECIMAL128I: { - batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { - auto string_val = column.stringval(); - StringParser::ParseResult result; - auto int128_val = StringParser::string_to_int( - string_val.c_str(), string_val.length(), &result); - DCHECK(result == StringParser::PARSE_SUCCESS); - set->insert(&int128_val); - }); - break; - } - case TYPE_DECIMAL256: { - batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { - auto string_val = column.stringval(); - StringParser::ParseResult result; - auto int_val = StringParser::string_to_int( - string_val.c_str(), string_val.length(), &result); - DCHECK(result == StringParser::PARSE_SUCCESS); - set->insert(&int_val); - }); - break; - } - case TYPE_VARCHAR: - case TYPE_CHAR: - case TYPE_STRING: { - batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { - const std::string& string_value = column.stringval(); - // string_value is std::string, call insert(data, size) function in StringSet will not cast as StringRef - // so could avoid some cast error at different class object. - set->insert((void*)string_value.data(), string_value.size()); - }); - break; - } - case TYPE_IPV4: { - batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { - int32_t tmp = column.intval(); - set->insert(&tmp); - }); - break; - } - case TYPE_IPV6: { - batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { - auto string_val = column.stringval(); - StringParser::ParseResult result; - auto int128_val = StringParser::string_to_int( - string_val.c_str(), string_val.length(), &result); - DCHECK(result == StringParser::PARSE_SUCCESS); - set->insert(&int128_val); - }); - break; - } - default: { - return Status::InternalError("not support assign to in filter, type: " + - type_to_string(_column_return_type)); - } - } - return Status::OK(); - } - - void set_enable_fixed_len_to_uint32_v2() { - if (_context->bloom_filter_func) { - _context->bloom_filter_func->set_enable_fixed_len_to_uint32_v2(); - } - } - - // used by shuffle runtime filter - // assign this filter by protobuf - Status assign(const PBloomFilter* bloom_filter, butil::IOBufAsZeroCopyInputStream* data, - bool contain_null) { - // we won't use this class to insert or find any data - // so any type is ok - _context->bloom_filter_func.reset(create_bloom_filter(_column_return_type == INVALID_TYPE - ? PrimitiveType::TYPE_INT - : _column_return_type)); - RETURN_IF_ERROR(_context->bloom_filter_func->assign(data, bloom_filter->filter_length(), - contain_null)); - return Status::OK(); - } - - // used by shuffle runtime filter - // assign this filter by protobuf - Status assign(const PMinMaxFilter* minmax_filter, bool contain_null) { - _context->minmax_func.reset(create_minmax_filter(_column_return_type)); - - if (contain_null) { - _context->minmax_func->set_null_aware(true); - _context->minmax_func->set_contain_null(); - } - - switch (_column_return_type) { - case TYPE_BOOLEAN: { - bool min_val = minmax_filter->min_val().boolval(); - bool max_val = minmax_filter->max_val().boolval(); - return _context->minmax_func->assign(&min_val, &max_val); - } - case TYPE_TINYINT: { - auto min_val = static_cast(minmax_filter->min_val().intval()); - auto max_val = static_cast(minmax_filter->max_val().intval()); - return _context->minmax_func->assign(&min_val, &max_val); - } - case TYPE_SMALLINT: { - auto min_val = static_cast(minmax_filter->min_val().intval()); - auto max_val = static_cast(minmax_filter->max_val().intval()); - return _context->minmax_func->assign(&min_val, &max_val); - } - case TYPE_INT: { - int32_t min_val = minmax_filter->min_val().intval(); - int32_t max_val = minmax_filter->max_val().intval(); - return _context->minmax_func->assign(&min_val, &max_val); - } - case TYPE_BIGINT: { - int64_t min_val = minmax_filter->min_val().longval(); - int64_t max_val = minmax_filter->max_val().longval(); - return _context->minmax_func->assign(&min_val, &max_val); - } - case TYPE_LARGEINT: { - auto min_string_val = minmax_filter->min_val().stringval(); - auto max_string_val = minmax_filter->max_val().stringval(); - StringParser::ParseResult result; - auto min_val = StringParser::string_to_int(min_string_val.c_str(), - min_string_val.length(), &result); - DCHECK(result == StringParser::PARSE_SUCCESS); - auto max_val = StringParser::string_to_int(max_string_val.c_str(), - max_string_val.length(), &result); - DCHECK(result == StringParser::PARSE_SUCCESS); - return _context->minmax_func->assign(&min_val, &max_val); - } - case TYPE_FLOAT: { - auto min_val = static_cast(minmax_filter->min_val().doubleval()); - auto max_val = static_cast(minmax_filter->max_val().doubleval()); - return _context->minmax_func->assign(&min_val, &max_val); - } - case TYPE_DOUBLE: { - auto min_val = static_cast(minmax_filter->min_val().doubleval()); - auto max_val = static_cast(minmax_filter->max_val().doubleval()); - return _context->minmax_func->assign(&min_val, &max_val); - } - case TYPE_DATEV2: { - int32_t min_val = minmax_filter->min_val().intval(); - int32_t max_val = minmax_filter->max_val().intval(); - return _context->minmax_func->assign(&min_val, &max_val); - } - case TYPE_DATETIMEV2: { - int64_t min_val = minmax_filter->min_val().longval(); - int64_t max_val = minmax_filter->max_val().longval(); - return _context->minmax_func->assign(&min_val, &max_val); - } - case TYPE_DATETIME: - case TYPE_DATE: { - const auto& min_val_ref = minmax_filter->min_val().stringval(); - const auto& max_val_ref = minmax_filter->max_val().stringval(); - VecDateTimeValue min_val; - VecDateTimeValue max_val; - min_val.from_date_str(min_val_ref.c_str(), min_val_ref.length()); - max_val.from_date_str(max_val_ref.c_str(), max_val_ref.length()); - return _context->minmax_func->assign(&min_val, &max_val); - } - case TYPE_DECIMALV2: { - const auto& min_val_ref = minmax_filter->min_val().stringval(); - const auto& max_val_ref = minmax_filter->max_val().stringval(); - DecimalV2Value min_val(min_val_ref); - DecimalV2Value max_val(max_val_ref); - return _context->minmax_func->assign(&min_val, &max_val); - } - case TYPE_DECIMAL32: { - int32_t min_val = minmax_filter->min_val().intval(); - int32_t max_val = minmax_filter->max_val().intval(); - return _context->minmax_func->assign(&min_val, &max_val); - } - case TYPE_DECIMAL64: { - int64_t min_val = minmax_filter->min_val().longval(); - int64_t max_val = minmax_filter->max_val().longval(); - return _context->minmax_func->assign(&min_val, &max_val); - } - case TYPE_DECIMAL128I: { - auto min_string_val = minmax_filter->min_val().stringval(); - auto max_string_val = minmax_filter->max_val().stringval(); - StringParser::ParseResult result; - auto min_val = StringParser::string_to_int(min_string_val.c_str(), - min_string_val.length(), &result); - DCHECK(result == StringParser::PARSE_SUCCESS); - auto max_val = StringParser::string_to_int(max_string_val.c_str(), - max_string_val.length(), &result); - DCHECK(result == StringParser::PARSE_SUCCESS); - return _context->minmax_func->assign(&min_val, &max_val); - } - case TYPE_DECIMAL256: { - auto min_string_val = minmax_filter->min_val().stringval(); - auto max_string_val = minmax_filter->max_val().stringval(); - StringParser::ParseResult result; - auto min_val = StringParser::string_to_int( - min_string_val.c_str(), min_string_val.length(), &result); - DCHECK(result == StringParser::PARSE_SUCCESS); - auto max_val = StringParser::string_to_int( - max_string_val.c_str(), max_string_val.length(), &result); - DCHECK(result == StringParser::PARSE_SUCCESS); - return _context->minmax_func->assign(&min_val, &max_val); - } - case TYPE_VARCHAR: - case TYPE_CHAR: - case TYPE_STRING: { - auto min_val_ref = minmax_filter->min_val().stringval(); - auto max_val_ref = minmax_filter->max_val().stringval(); - return _context->minmax_func->assign(&min_val_ref, &max_val_ref); - } - case TYPE_IPV4: { - int tmp_min = minmax_filter->min_val().intval(); - int tmp_max = minmax_filter->max_val().intval(); - return _context->minmax_func->assign(&tmp_min, &tmp_max); - } - case TYPE_IPV6: { - auto min_string_val = minmax_filter->min_val().stringval(); - auto max_string_val = minmax_filter->max_val().stringval(); - StringParser::ParseResult result; - auto min_val = StringParser::string_to_int(min_string_val.c_str(), - min_string_val.length(), &result); - DCHECK(result == StringParser::PARSE_SUCCESS); - auto max_val = StringParser::string_to_int(max_string_val.c_str(), - max_string_val.length(), &result); - DCHECK(result == StringParser::PARSE_SUCCESS); - return _context->minmax_func->assign(&min_val, &max_val); - } - default: - break; - } - return Status::InternalError("not support!"); - } - - void get_bloom_filter_desc(char** data, int* filter_length) { - _context->bloom_filter_func->get_data(data, filter_length); - } - - PrimitiveType column_type() { return _column_return_type; } - - bool is_bloomfilter() const { return get_real_type() == RuntimeFilterType::BLOOM_FILTER; } - - bool contain_null() const { - if (is_bloomfilter()) { - return _context->bloom_filter_func->contain_null(); - } - if (_context->hybrid_set) { - if (get_real_type() != RuntimeFilterType::IN_FILTER) { - throw Exception(ErrorCode::INTERNAL_ERROR, "rf has hybrid_set but real type is {}", - int(get_real_type())); - } - return _context->hybrid_set->contain_null(); - } - if (_context->minmax_func) { - return _context->minmax_func->contain_null(); - } - return false; - } - - bool is_ignored() const { return _context->ignored; } - - void set_ignored() { _context->ignored = true; } - - bool is_disabled() const { return _context->disabled; } - - void set_disabled() { - _context->disabled = true; - _context->minmax_func.reset(); - _context->hybrid_set.reset(); - _context->bloom_filter_func.reset(); - _context->bitmap_filter_func.reset(); - } - - void batch_assign(const PInFilter* filter, - void (*assign_func)(std::shared_ptr& _hybrid_set, - PColumnValue&)) { - for (int i = 0; i < filter->values_size(); ++i) { - PColumnValue column = filter->values(i); - assign_func(_context->hybrid_set, column); - } - } - - size_t get_in_filter_size() const { - return _context->hybrid_set ? _context->hybrid_set->size() : 0; - } - - std::shared_ptr get_bitmap_filter() const { - return _context->bitmap_filter_func; - } - - friend class IRuntimeFilter; - - void set_filter_id(int id) { - if (_context->bloom_filter_func) { - _context->bloom_filter_func->set_filter_id(id); - } - if (_context->bitmap_filter_func) { - _context->bitmap_filter_func->set_filter_id(id); - } - if (_context->hybrid_set) { - _context->hybrid_set->set_filter_id(id); - } - } - -private: - // When a runtime filter received from remote and it is a bloom filter, _column_return_type will be invalid. - PrimitiveType _column_return_type; // column type - RuntimeFilterType _filter_type; - int32_t _max_in_num = -1; - - RuntimeFilterContextSPtr _context; - uint32_t _filter_id; -}; Status IRuntimeFilter::create(RuntimeFilterParamsContext* state, const TRuntimeFilterDesc* desc, const TQueryOptions* query_options, const RuntimeFilterRole role, @@ -1572,11 +867,12 @@ void IRuntimeFilter::update_runtime_filter_type_to_profile(uint64_t local_merge_ std::string IRuntimeFilter::debug_string() const { return fmt::format( - "RuntimeFilter: (id = {}, type = {}, is_broadcast: {}, ignored: {}, " + "RuntimeFilter: (id = {}, type = {}, is_broadcast: {}, ignored: {}, disabled: {}, " "build_bf_cardinality: {}, dependency: {}, synced_size: {}, has_local_target: {}, " "has_remote_target: {}, error_msg: [{}]", _filter_id, to_string(_runtime_filter_type), _is_broadcast_join, - _wrapper->_context->ignored, _wrapper->get_build_bf_cardinality(), + _wrapper->_context->ignored, _wrapper->_context->disabled, + _wrapper->get_build_bf_cardinality(), _dependency ? _dependency->debug_string() : "none", _synced_size, _has_local_target, _has_remote_target, _wrapper->_context->err_msg); } @@ -1794,4 +1090,638 @@ Status RuntimePredicateWrapper::get_push_exprs( RuntimeFilterWrapperHolder::RuntimeFilterWrapperHolder() = default; RuntimeFilterWrapperHolder::~RuntimeFilterWrapperHolder() = default; +Status RuntimePredicateWrapper::init(const RuntimeFilterParams* params) { + _max_in_num = params->max_in_num; + switch (_filter_type) { + case RuntimeFilterType::IN_FILTER: { + _context->hybrid_set.reset(create_set(_column_return_type)); + _context->hybrid_set->set_null_aware(params->null_aware); + break; + } + // Only use in nested loop join not need set null aware + case RuntimeFilterType::MIN_FILTER: + case RuntimeFilterType::MAX_FILTER: { + _context->minmax_func.reset(create_minmax_filter(_column_return_type)); + break; + } + case RuntimeFilterType::MINMAX_FILTER: { + _context->minmax_func.reset(create_minmax_filter(_column_return_type)); + _context->minmax_func->set_null_aware(params->null_aware); + break; + } + case RuntimeFilterType::BLOOM_FILTER: { + _context->bloom_filter_func.reset(create_bloom_filter(_column_return_type)); + _context->bloom_filter_func->init_params(params); + return Status::OK(); + } + case RuntimeFilterType::IN_OR_BLOOM_FILTER: { + _context->hybrid_set.reset(create_set(_column_return_type)); + _context->hybrid_set->set_null_aware(params->null_aware); + _context->bloom_filter_func.reset(create_bloom_filter(_column_return_type)); + _context->bloom_filter_func->init_params(params); + return Status::OK(); + } + case RuntimeFilterType::BITMAP_FILTER: { + _context->bitmap_filter_func.reset(create_bitmap_filter(_column_return_type)); + _context->bitmap_filter_func->set_not_in(params->bitmap_filter_not_in); + return Status::OK(); + } + default: + return Status::InternalError("Unknown Filter type"); + } + return Status::OK(); +} + +Status RuntimePredicateWrapper::change_to_bloom_filter() { + if (_filter_type != RuntimeFilterType::IN_OR_BLOOM_FILTER) { + return Status::InternalError( + "Can not change to bloom filter because of runtime filter type is {}", + IRuntimeFilter::to_string(_filter_type)); + } + BloomFilterFuncBase* bf = _context->bloom_filter_func.get(); + + if (bf != nullptr) { + insert_to_bloom_filter(bf); + } else if (_context->hybrid_set != nullptr && _context->hybrid_set->size() != 0) { + return Status::InternalError("change to bloom filter need empty set ", + IRuntimeFilter::to_string(_filter_type)); + } + + // release in filter + _context->hybrid_set.reset(); + return Status::OK(); +} + +void RuntimePredicateWrapper::set_filter_id(int id) { + if (_context->bloom_filter_func) { + _context->bloom_filter_func->set_filter_id(id); + } + if (_context->bitmap_filter_func) { + _context->bitmap_filter_func->set_filter_id(id); + } + if (_context->hybrid_set) { + _context->hybrid_set->set_filter_id(id); + } +} + +void RuntimePredicateWrapper::batch_assign( + const PInFilter* filter, + void (*assign_func)(std::shared_ptr& _hybrid_set, PColumnValue&)) { + for (int i = 0; i < filter->values_size(); ++i) { + PColumnValue column = filter->values(i); + assign_func(_context->hybrid_set, column); + } +} + +Status RuntimePredicateWrapper::init_bloom_filter(const size_t build_bf_cardinality) { + if (_filter_type != RuntimeFilterType::BLOOM_FILTER && + _filter_type != RuntimeFilterType::IN_OR_BLOOM_FILTER) { + throw Exception(ErrorCode::INTERNAL_ERROR, "init_bloom_filter meet invalid input type {}", + int(_filter_type)); + } + return _context->bloom_filter_func->init_with_cardinality(build_bf_cardinality); +} + +bool RuntimePredicateWrapper::get_build_bf_cardinality() const { + return _context->bloom_filter_func && _context->bloom_filter_func->get_build_bf_cardinality(); +} + +void RuntimePredicateWrapper::insert_to_bloom_filter(BloomFilterFuncBase* bloom_filter) const { + if (_context->hybrid_set->size() > 0) { + auto* it = _context->hybrid_set->begin(); + while (it->has_next()) { + bloom_filter->insert(it->get_value()); + it->next(); + } + } + if (_context->hybrid_set->contain_null()) { + bloom_filter->set_contain_null_and_null_aware(); + } +} + +void RuntimePredicateWrapper::insert_fixed_len(const vectorized::ColumnPtr& column, size_t start) { + if (is_ignored()) { + throw Exception(ErrorCode::INTERNAL_ERROR, "insert_fixed_len meet ignored rf"); + } + switch (_filter_type) { + case RuntimeFilterType::IN_FILTER: { + _context->hybrid_set->insert_fixed_len(column, start); + break; + } + case RuntimeFilterType::MIN_FILTER: + case RuntimeFilterType::MAX_FILTER: + case RuntimeFilterType::MINMAX_FILTER: { + _context->minmax_func->insert_fixed_len(column, start); + break; + } + case RuntimeFilterType::BLOOM_FILTER: { + _context->bloom_filter_func->insert_fixed_len(column, start); + break; + } + case RuntimeFilterType::IN_OR_BLOOM_FILTER: { + if (is_bloomfilter()) { + _context->bloom_filter_func->insert_fixed_len(column, start); + } else { + _context->hybrid_set->insert_fixed_len(column, start); + } + break; + } + default: + DCHECK(false); + break; + } +} + +void RuntimePredicateWrapper::bitmap_filter_insert_batch(const vectorized::ColumnPtr column, + size_t start) { + std::vector bitmaps; + if (column->is_nullable()) { + const auto* nullable = assert_cast(column.get()); + const auto& col = + assert_cast(nullable->get_nested_column()); + const auto& nullmap = + assert_cast(nullable->get_null_map_column()) + .get_data(); + for (size_t i = start; i < column->size(); i++) { + if (!nullmap[i]) { + bitmaps.push_back(&(col.get_data()[i])); + } + } + } else { + const auto* col = assert_cast(column.get()); + for (size_t i = start; i < column->size(); i++) { + bitmaps.push_back(&(col->get_data()[i])); + } + } + _context->bitmap_filter_func->insert_many(bitmaps); +} + +size_t RuntimePredicateWrapper::get_bloom_filter_size() const { + return _context->bloom_filter_func ? _context->bloom_filter_func->get_size() : 0; +} + +Status RuntimePredicateWrapper::merge(const RuntimePredicateWrapper* wrapper) { + if (wrapper->is_disabled()) { + set_disabled(); + return Status::OK(); + } + + if (wrapper->is_ignored() || is_disabled()) { + return Status::OK(); + } + + _context->ignored = false; + + bool can_not_merge_in_or_bloom = + _filter_type == RuntimeFilterType::IN_OR_BLOOM_FILTER && + (wrapper->_filter_type != RuntimeFilterType::IN_FILTER && + wrapper->_filter_type != RuntimeFilterType::BLOOM_FILTER && + wrapper->_filter_type != RuntimeFilterType::IN_OR_BLOOM_FILTER); + + bool can_not_merge_other = _filter_type != RuntimeFilterType::IN_OR_BLOOM_FILTER && + _filter_type != wrapper->_filter_type; + + CHECK(!can_not_merge_in_or_bloom && !can_not_merge_other) + << " can not merge runtime filter(id=" << _filter_id << "), current is filter type is " + << IRuntimeFilter::to_string(_filter_type) << ", other filter type is " + << IRuntimeFilter::to_string(wrapper->_filter_type); + + switch (_filter_type) { + case RuntimeFilterType::IN_FILTER: { + _context->hybrid_set->insert(wrapper->_context->hybrid_set.get()); + if (_max_in_num >= 0 && _context->hybrid_set->size() >= _max_in_num) { + set_disabled(); + } + break; + } + case RuntimeFilterType::MIN_FILTER: + case RuntimeFilterType::MAX_FILTER: + case RuntimeFilterType::MINMAX_FILTER: { + RETURN_IF_ERROR(_context->minmax_func->merge(wrapper->_context->minmax_func.get())); + break; + } + case RuntimeFilterType::BLOOM_FILTER: { + RETURN_IF_ERROR( + _context->bloom_filter_func->merge(wrapper->_context->bloom_filter_func.get())); + break; + } + case RuntimeFilterType::IN_OR_BLOOM_FILTER: { + auto real_filter_type = get_real_type(); + + auto other_filter_type = wrapper->_filter_type; + if (other_filter_type == RuntimeFilterType::IN_OR_BLOOM_FILTER) { + other_filter_type = wrapper->get_real_type(); + } + + if (real_filter_type == RuntimeFilterType::IN_FILTER) { + // when we meet base rf is in-filter, threre only have two case: + // case1: all input-filter's build_bf_exactly is true, inited by synced global size + // case2: all input-filter's build_bf_exactly is false, inited by default size + if (other_filter_type == RuntimeFilterType::IN_FILTER) { + _context->hybrid_set->insert(wrapper->_context->hybrid_set.get()); + if (_max_in_num >= 0 && _context->hybrid_set->size() >= _max_in_num) { + // case2: use default size to init bf + RETURN_IF_ERROR(_context->bloom_filter_func->init_with_fixed_length()); + RETURN_IF_ERROR(change_to_bloom_filter()); + } + } else { + // case1&case2: use input bf directly and insert hybrid set data into bf + _context->bloom_filter_func = wrapper->_context->bloom_filter_func; + RETURN_IF_ERROR(change_to_bloom_filter()); + } + } else { + if (other_filter_type == RuntimeFilterType::IN_FILTER) { + // case2: insert data to global filter + wrapper->insert_to_bloom_filter(_context->bloom_filter_func.get()); + } else { + // case1&case2: all input bf must has same size + RETURN_IF_ERROR(_context->bloom_filter_func->merge( + wrapper->_context->bloom_filter_func.get())); + } + } + break; + } + case RuntimeFilterType::BITMAP_FILTER: { + // use input bitmap directly because we assume bitmap filter join always have full data + _context->bitmap_filter_func = wrapper->_context->bitmap_filter_func; + break; + } + default: + return Status::InternalError("unknown runtime filter"); + } + return Status::OK(); +} + +Status RuntimePredicateWrapper::assign(const PInFilter* in_filter, bool contain_null) { + _context->hybrid_set.reset(create_set(_column_return_type)); + if (contain_null) { + _context->hybrid_set->set_null_aware(true); + _context->hybrid_set->insert((const void*)nullptr); + } + + switch (_column_return_type) { + case TYPE_BOOLEAN: { + batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { + bool bool_val = column.boolval(); + set->insert(&bool_val); + }); + break; + } + case TYPE_TINYINT: { + batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { + auto int_val = static_cast(column.intval()); + set->insert(&int_val); + }); + break; + } + case TYPE_SMALLINT: { + batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { + auto int_val = static_cast(column.intval()); + set->insert(&int_val); + }); + break; + } + case TYPE_INT: { + batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { + int32_t int_val = column.intval(); + set->insert(&int_val); + }); + break; + } + case TYPE_BIGINT: { + batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { + int64_t long_val = column.longval(); + set->insert(&long_val); + }); + break; + } + case TYPE_LARGEINT: { + batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { + auto string_val = column.stringval(); + StringParser::ParseResult result; + auto int128_val = StringParser::string_to_int(string_val.c_str(), + string_val.length(), &result); + DCHECK(result == StringParser::PARSE_SUCCESS); + set->insert(&int128_val); + }); + break; + } + case TYPE_FLOAT: { + batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { + auto float_val = static_cast(column.doubleval()); + set->insert(&float_val); + }); + break; + } + case TYPE_DOUBLE: { + batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { + double double_val = column.doubleval(); + set->insert(&double_val); + }); + break; + } + case TYPE_DATEV2: { + batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { + auto date_v2_val = column.intval(); + set->insert(&date_v2_val); + }); + break; + } + case TYPE_DATETIMEV2: { + batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { + auto date_v2_val = column.longval(); + set->insert(&date_v2_val); + }); + break; + } + case TYPE_DATETIME: + case TYPE_DATE: { + batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { + const auto& string_val_ref = column.stringval(); + VecDateTimeValue datetime_val; + datetime_val.from_date_str(string_val_ref.c_str(), string_val_ref.length()); + set->insert(&datetime_val); + }); + break; + } + case TYPE_DECIMALV2: { + batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { + const auto& string_val_ref = column.stringval(); + DecimalV2Value decimal_val(string_val_ref); + set->insert(&decimal_val); + }); + break; + } + case TYPE_DECIMAL32: { + batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { + int32_t decimal_32_val = column.intval(); + set->insert(&decimal_32_val); + }); + break; + } + case TYPE_DECIMAL64: { + batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { + int64_t decimal_64_val = column.longval(); + set->insert(&decimal_64_val); + }); + break; + } + case TYPE_DECIMAL128I: { + batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { + auto string_val = column.stringval(); + StringParser::ParseResult result; + auto int128_val = StringParser::string_to_int(string_val.c_str(), + string_val.length(), &result); + DCHECK(result == StringParser::PARSE_SUCCESS); + set->insert(&int128_val); + }); + break; + } + case TYPE_DECIMAL256: { + batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { + auto string_val = column.stringval(); + StringParser::ParseResult result; + auto int_val = StringParser::string_to_int(string_val.c_str(), + string_val.length(), &result); + DCHECK(result == StringParser::PARSE_SUCCESS); + set->insert(&int_val); + }); + break; + } + case TYPE_VARCHAR: + case TYPE_CHAR: + case TYPE_STRING: { + batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { + const std::string& string_value = column.stringval(); + // string_value is std::string, call insert(data, size) function in StringSet will not cast as StringRef + // so could avoid some cast error at different class object. + set->insert((void*)string_value.data(), string_value.size()); + }); + break; + } + case TYPE_IPV4: { + batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { + int32_t tmp = column.intval(); + set->insert(&tmp); + }); + break; + } + case TYPE_IPV6: { + batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { + auto string_val = column.stringval(); + StringParser::ParseResult result; + auto int128_val = StringParser::string_to_int(string_val.c_str(), + string_val.length(), &result); + DCHECK(result == StringParser::PARSE_SUCCESS); + set->insert(&int128_val); + }); + break; + } + default: { + return Status::InternalError("not support assign to in filter, type: " + + type_to_string(_column_return_type)); + } + } + return Status::OK(); +} + +void RuntimePredicateWrapper::set_enable_fixed_len_to_uint32_v2() { + if (_context->bloom_filter_func) { + _context->bloom_filter_func->set_enable_fixed_len_to_uint32_v2(); + } +} + +Status RuntimePredicateWrapper::assign(const PBloomFilter* bloom_filter, + butil::IOBufAsZeroCopyInputStream* data, bool contain_null) { + // we won't use this class to insert or find any data + // so any type is ok + _context->bloom_filter_func.reset(create_bloom_filter( + _column_return_type == INVALID_TYPE ? PrimitiveType::TYPE_INT : _column_return_type)); + RETURN_IF_ERROR( + _context->bloom_filter_func->assign(data, bloom_filter->filter_length(), contain_null)); + return Status::OK(); +} + +// used by shuffle runtime filter +// assign this filter by protobuf +Status RuntimePredicateWrapper::assign(const PMinMaxFilter* minmax_filter, bool contain_null) { + _context->minmax_func.reset(create_minmax_filter(_column_return_type)); + + if (contain_null) { + _context->minmax_func->set_null_aware(true); + _context->minmax_func->set_contain_null(); + } + + switch (_column_return_type) { + case TYPE_BOOLEAN: { + bool min_val = minmax_filter->min_val().boolval(); + bool max_val = minmax_filter->max_val().boolval(); + return _context->minmax_func->assign(&min_val, &max_val); + } + case TYPE_TINYINT: { + auto min_val = static_cast(minmax_filter->min_val().intval()); + auto max_val = static_cast(minmax_filter->max_val().intval()); + return _context->minmax_func->assign(&min_val, &max_val); + } + case TYPE_SMALLINT: { + auto min_val = static_cast(minmax_filter->min_val().intval()); + auto max_val = static_cast(minmax_filter->max_val().intval()); + return _context->minmax_func->assign(&min_val, &max_val); + } + case TYPE_INT: { + int32_t min_val = minmax_filter->min_val().intval(); + int32_t max_val = minmax_filter->max_val().intval(); + return _context->minmax_func->assign(&min_val, &max_val); + } + case TYPE_BIGINT: { + int64_t min_val = minmax_filter->min_val().longval(); + int64_t max_val = minmax_filter->max_val().longval(); + return _context->minmax_func->assign(&min_val, &max_val); + } + case TYPE_LARGEINT: { + auto min_string_val = minmax_filter->min_val().stringval(); + auto max_string_val = minmax_filter->max_val().stringval(); + StringParser::ParseResult result; + auto min_val = StringParser::string_to_int(min_string_val.c_str(), + min_string_val.length(), &result); + DCHECK(result == StringParser::PARSE_SUCCESS); + auto max_val = StringParser::string_to_int(max_string_val.c_str(), + max_string_val.length(), &result); + DCHECK(result == StringParser::PARSE_SUCCESS); + return _context->minmax_func->assign(&min_val, &max_val); + } + case TYPE_FLOAT: { + auto min_val = static_cast(minmax_filter->min_val().doubleval()); + auto max_val = static_cast(minmax_filter->max_val().doubleval()); + return _context->minmax_func->assign(&min_val, &max_val); + } + case TYPE_DOUBLE: { + auto min_val = static_cast(minmax_filter->min_val().doubleval()); + auto max_val = static_cast(minmax_filter->max_val().doubleval()); + return _context->minmax_func->assign(&min_val, &max_val); + } + case TYPE_DATEV2: { + int32_t min_val = minmax_filter->min_val().intval(); + int32_t max_val = minmax_filter->max_val().intval(); + return _context->minmax_func->assign(&min_val, &max_val); + } + case TYPE_DATETIMEV2: { + int64_t min_val = minmax_filter->min_val().longval(); + int64_t max_val = minmax_filter->max_val().longval(); + return _context->minmax_func->assign(&min_val, &max_val); + } + case TYPE_DATETIME: + case TYPE_DATE: { + const auto& min_val_ref = minmax_filter->min_val().stringval(); + const auto& max_val_ref = minmax_filter->max_val().stringval(); + VecDateTimeValue min_val; + VecDateTimeValue max_val; + min_val.from_date_str(min_val_ref.c_str(), min_val_ref.length()); + max_val.from_date_str(max_val_ref.c_str(), max_val_ref.length()); + return _context->minmax_func->assign(&min_val, &max_val); + } + case TYPE_DECIMALV2: { + const auto& min_val_ref = minmax_filter->min_val().stringval(); + const auto& max_val_ref = minmax_filter->max_val().stringval(); + DecimalV2Value min_val(min_val_ref); + DecimalV2Value max_val(max_val_ref); + return _context->minmax_func->assign(&min_val, &max_val); + } + case TYPE_DECIMAL32: { + int32_t min_val = minmax_filter->min_val().intval(); + int32_t max_val = minmax_filter->max_val().intval(); + return _context->minmax_func->assign(&min_val, &max_val); + } + case TYPE_DECIMAL64: { + int64_t min_val = minmax_filter->min_val().longval(); + int64_t max_val = minmax_filter->max_val().longval(); + return _context->minmax_func->assign(&min_val, &max_val); + } + case TYPE_DECIMAL128I: { + auto min_string_val = minmax_filter->min_val().stringval(); + auto max_string_val = minmax_filter->max_val().stringval(); + StringParser::ParseResult result; + auto min_val = StringParser::string_to_int(min_string_val.c_str(), + min_string_val.length(), &result); + DCHECK(result == StringParser::PARSE_SUCCESS); + auto max_val = StringParser::string_to_int(max_string_val.c_str(), + max_string_val.length(), &result); + DCHECK(result == StringParser::PARSE_SUCCESS); + return _context->minmax_func->assign(&min_val, &max_val); + } + case TYPE_DECIMAL256: { + auto min_string_val = minmax_filter->min_val().stringval(); + auto max_string_val = minmax_filter->max_val().stringval(); + StringParser::ParseResult result; + auto min_val = StringParser::string_to_int(min_string_val.c_str(), + min_string_val.length(), &result); + DCHECK(result == StringParser::PARSE_SUCCESS); + auto max_val = StringParser::string_to_int(max_string_val.c_str(), + max_string_val.length(), &result); + DCHECK(result == StringParser::PARSE_SUCCESS); + return _context->minmax_func->assign(&min_val, &max_val); + } + case TYPE_VARCHAR: + case TYPE_CHAR: + case TYPE_STRING: { + auto min_val_ref = minmax_filter->min_val().stringval(); + auto max_val_ref = minmax_filter->max_val().stringval(); + return _context->minmax_func->assign(&min_val_ref, &max_val_ref); + } + case TYPE_IPV4: { + int tmp_min = minmax_filter->min_val().intval(); + int tmp_max = minmax_filter->max_val().intval(); + return _context->minmax_func->assign(&tmp_min, &tmp_max); + } + case TYPE_IPV6: { + auto min_string_val = minmax_filter->min_val().stringval(); + auto max_string_val = minmax_filter->max_val().stringval(); + StringParser::ParseResult result; + auto min_val = StringParser::string_to_int(min_string_val.c_str(), + min_string_val.length(), &result); + DCHECK(result == StringParser::PARSE_SUCCESS); + auto max_val = StringParser::string_to_int(max_string_val.c_str(), + max_string_val.length(), &result); + DCHECK(result == StringParser::PARSE_SUCCESS); + return _context->minmax_func->assign(&min_val, &max_val); + } + default: + break; + } + return Status::InternalError("not support!"); +} + +void RuntimePredicateWrapper::get_bloom_filter_desc(char** data, int* filter_length) { + _context->bloom_filter_func->get_data(data, filter_length); +} + +bool RuntimePredicateWrapper::contain_null() const { + if (is_bloomfilter()) { + return _context->bloom_filter_func->contain_null(); + } + if (_context->hybrid_set) { + if (get_real_type() != RuntimeFilterType::IN_FILTER) { + throw Exception(ErrorCode::INTERNAL_ERROR, "rf has hybrid_set but real type is {}", + int(get_real_type())); + } + return _context->hybrid_set->contain_null(); + } + if (_context->minmax_func) { + return _context->minmax_func->contain_null(); + } + return false; +} + +size_t RuntimePredicateWrapper::get_in_filter_size() const { + return _context->hybrid_set ? _context->hybrid_set->size() : 0; +} + +void RuntimePredicateWrapper::set_disabled() { + _context->disabled = true; + _context->minmax_func.reset(); + _context->hybrid_set.reset(); + _context->bloom_filter_func.reset(); + _context->bitmap_filter_func.reset(); +} + } // namespace doris diff --git a/be/src/exprs/runtime_filter.h b/be/src/exprs/runtime_filter.h index 441de7d4da340c..e38f6901ef4618 100644 --- a/be/src/exprs/runtime_filter.h +++ b/be/src/exprs/runtime_filter.h @@ -443,4 +443,115 @@ class RuntimeFilterWrapperHolder { WrapperPtr _wrapper; }; +// This class is a wrapper of runtime predicate function +class RuntimePredicateWrapper { +public: + RuntimePredicateWrapper(const RuntimeFilterParams* params) + : RuntimePredicateWrapper(params->column_return_type, params->filter_type, + params->filter_id) {}; + // for a 'tmp' runtime predicate wrapper + // only could called assign method or as a param for merge + RuntimePredicateWrapper(PrimitiveType column_type, RuntimeFilterType type, uint32_t filter_id) + : _column_return_type(column_type), + _filter_type(type), + _context(new RuntimeFilterContext()), + _filter_id(filter_id) {} + + // init runtime filter wrapper + // alloc memory to init runtime filter function + Status init(const RuntimeFilterParams* params); + + Status change_to_bloom_filter(); + + Status init_bloom_filter(const size_t build_bf_cardinality); + + bool get_build_bf_cardinality() const; + + void insert_to_bloom_filter(BloomFilterFuncBase* bloom_filter) const; + + BloomFilterFuncBase* get_bloomfilter() const { return _context->bloom_filter_func.get(); } + + void insert_fixed_len(const vectorized::ColumnPtr& column, size_t start); + + void insert_batch(const vectorized::ColumnPtr& column, size_t start) { + if (get_real_type() == RuntimeFilterType::BITMAP_FILTER) { + bitmap_filter_insert_batch(column, start); + } else { + insert_fixed_len(column, start); + } + } + + void bitmap_filter_insert_batch(const vectorized::ColumnPtr column, size_t start); + + RuntimeFilterType get_real_type() const { + if (_filter_type == RuntimeFilterType::IN_OR_BLOOM_FILTER) { + if (_context->hybrid_set) { + return RuntimeFilterType::IN_FILTER; + } + return RuntimeFilterType::BLOOM_FILTER; + } + return _filter_type; + } + + size_t get_bloom_filter_size() const; + + Status get_push_exprs(std::list& probe_ctxs, + std::vector& push_exprs, + const TExpr& probe_expr); + + Status merge(const RuntimePredicateWrapper* wrapper); + + Status assign(const PInFilter* in_filter, bool contain_null); + + void set_enable_fixed_len_to_uint32_v2(); + + // used by shuffle runtime filter + // assign this filter by protobuf + Status assign(const PBloomFilter* bloom_filter, butil::IOBufAsZeroCopyInputStream* data, + bool contain_null); + + // used by shuffle runtime filter + // assign this filter by protobuf + Status assign(const PMinMaxFilter* minmax_filter, bool contain_null); + + void get_bloom_filter_desc(char** data, int* filter_length); + + PrimitiveType column_type() { return _column_return_type; } + + bool is_bloomfilter() const { return get_real_type() == RuntimeFilterType::BLOOM_FILTER; } + + bool contain_null() const; + + bool is_ignored() const { return _context->ignored; } + + void set_ignored() { _context->ignored = true; } + + bool is_disabled() const { return _context->disabled; } + + void set_disabled(); + + void batch_assign(const PInFilter* filter, + void (*assign_func)(std::shared_ptr& _hybrid_set, + PColumnValue&)); + + size_t get_in_filter_size() const; + + std::shared_ptr get_bitmap_filter() const { + return _context->bitmap_filter_func; + } + + friend class IRuntimeFilter; + + void set_filter_id(int id); + +private: + // When a runtime filter received from remote and it is a bloom filter, _column_return_type will be invalid. + PrimitiveType _column_return_type; // column type + RuntimeFilterType _filter_type; + int32_t _max_in_num = -1; + + RuntimeFilterContextSPtr _context; + uint32_t _filter_id; +}; + } // namespace doris diff --git a/be/src/exprs/runtime_filter_slots.h b/be/src/exprs/runtime_filter_slots.h index a9dd631e3581a2..b732ae155f7531 100644 --- a/be/src/exprs/runtime_filter_slots.h +++ b/be/src/exprs/runtime_filter_slots.h @@ -35,11 +35,7 @@ class VRuntimeFilterSlots { VRuntimeFilterSlots( const std::vector>& build_expr_ctxs, const std::vector>& runtime_filters) - : _build_expr_context(build_expr_ctxs), _runtime_filters(runtime_filters) { - for (auto runtime_filter : _runtime_filters) { - _runtime_filters_map[runtime_filter->expr_order()].push_back(runtime_filter.get()); - } - } + : _build_expr_context(build_expr_ctxs), _runtime_filters(runtime_filters) {} Status send_filter_size(RuntimeState* state, uint64_t hash_table_size, std::shared_ptr dependency) { @@ -139,62 +135,48 @@ class VRuntimeFilterSlots { } void insert(const vectorized::Block* block) { - for (int i = 0; i < _build_expr_context.size(); ++i) { - auto iter = _runtime_filters_map.find(i); - if (iter == _runtime_filters_map.end()) { - continue; - } - - int result_column_id = _build_expr_context[i]->get_last_result_column_id(); + for (auto& filter : _runtime_filters) { + int result_column_id = + _build_expr_context[filter->expr_order()]->get_last_result_column_id(); const auto& column = block->get_by_position(result_column_id).column; - for (auto* filter : iter->second) { - if (filter->get_ignored() || filter->get_disabled()) { - continue; - } - filter->insert_batch(column, 1); + if (filter->get_ignored() || filter->get_disabled()) { + continue; } + filter->insert_batch(column, 1); } } // publish runtime filter Status publish(RuntimeState* state, bool publish_local) { - for (auto& pair : _runtime_filters_map) { - for (auto& filter : pair.second) { - RETURN_IF_ERROR(filter->publish(state, publish_local)); - } + for (auto& filter : _runtime_filters) { + RETURN_IF_ERROR(filter->publish(state, publish_local)); } return Status::OK(); } void copy_to_shared_context(vectorized::SharedHashTableContextPtr& context) { - for (auto& it : _runtime_filters_map) { - for (auto& filter : it.second) { - context->runtime_filters[filter->filter_id()] = filter->get_shared_context_ref(); - } + for (auto& filter : _runtime_filters) { + context->runtime_filters[filter->filter_id()] = filter->get_shared_context_ref(); } } Status copy_from_shared_context(vectorized::SharedHashTableContextPtr& context) { - for (auto& it : _runtime_filters_map) { - for (auto& filter : it.second) { - auto filter_id = filter->filter_id(); - auto ret = context->runtime_filters.find(filter_id); - if (ret == context->runtime_filters.end()) { - return Status::Aborted("invalid runtime filter id: {}", filter_id); - } - filter->get_shared_context_ref() = ret->second; + for (auto& filter : _runtime_filters) { + auto filter_id = filter->filter_id(); + auto ret = context->runtime_filters.find(filter_id); + if (ret == context->runtime_filters.end()) { + return Status::Aborted("invalid runtime filter id: {}", filter_id); } + filter->get_shared_context_ref() = ret->second; } return Status::OK(); } - bool empty() { return _runtime_filters_map.empty(); } + bool empty() { return _runtime_filters.empty(); } private: const std::vector>& _build_expr_context; std::vector> _runtime_filters; - // prob_contition index -> [IRuntimeFilter] - std::map> _runtime_filters_map; }; } // namespace doris diff --git a/be/src/io/cache/cached_remote_file_reader.cpp b/be/src/io/cache/cached_remote_file_reader.cpp index 70765fa707ea87..d1bd0b8023c23e 100644 --- a/be/src/io/cache/cached_remote_file_reader.cpp +++ b/be/src/io/cache/cached_remote_file_reader.cpp @@ -126,8 +126,12 @@ Status CachedRemoteFileReader::read_at_impl(size_t offset, Slice result, size_t* ReadStatistics stats; auto defer_func = [&](int*) { if (io_ctx->file_cache_stats) { - _update_state(stats, io_ctx->file_cache_stats, io_ctx->is_inverted_index); - io::FileCacheProfile::instance().update(io_ctx->file_cache_stats); + // update stats in io_ctx, for query profile + _update_stats(stats, io_ctx->file_cache_stats, io_ctx->is_inverted_index); + // update stats increment in this reading procedure for file cache metrics + FileCacheStatistics fcache_stats_increment; + _update_stats(stats, &fcache_stats_increment, io_ctx->is_inverted_index); + io::FileCacheProfile::instance().update(&fcache_stats_increment); } }; std::unique_ptr defer((int*)0x01, std::move(defer_func)); @@ -316,7 +320,7 @@ Status CachedRemoteFileReader::read_at_impl(size_t offset, Slice result, size_t* return Status::OK(); } -void CachedRemoteFileReader::_update_state(const ReadStatistics& read_stats, +void CachedRemoteFileReader::_update_stats(const ReadStatistics& read_stats, FileCacheStatistics* statis, bool is_inverted_index) const { if (statis == nullptr) { diff --git a/be/src/io/cache/cached_remote_file_reader.h b/be/src/io/cache/cached_remote_file_reader.h index 735e652f94cadc..94e8a5807ba273 100644 --- a/be/src/io/cache/cached_remote_file_reader.h +++ b/be/src/io/cache/cached_remote_file_reader.h @@ -67,7 +67,7 @@ class CachedRemoteFileReader final : public FileReader { std::shared_mutex _mtx; std::map _cache_file_readers; - void _update_state(const ReadStatistics& stats, FileCacheStatistics* state, + void _update_stats(const ReadStatistics& stats, FileCacheStatistics* state, bool is_inverted_index) const; }; diff --git a/be/src/pipeline/exec/operator.cpp b/be/src/pipeline/exec/operator.cpp index bb254aae72b8a7..9ca0f0fcd40dd6 100644 --- a/be/src/pipeline/exec/operator.cpp +++ b/be/src/pipeline/exec/operator.cpp @@ -295,7 +295,7 @@ Status OperatorXBase::do_projections(RuntimeState* state, vectorized::Block* ori *_output_row_descriptor); if (rows != 0) { auto& mutable_columns = mutable_block.mutable_columns(); - DCHECK(mutable_columns.size() == local_state->_projections.size()); + DCHECK_EQ(mutable_columns.size(), local_state->_projections.size()) << debug_string(); for (int i = 0; i < mutable_columns.size(); ++i) { auto result_column_id = -1; RETURN_IF_ERROR(local_state->_projections[i]->execute(&input_block, &result_column_id)); diff --git a/be/src/pipeline/pipeline_fragment_context.cpp b/be/src/pipeline/pipeline_fragment_context.cpp index e8e8ed5d9fea05..c8e3429107c507 100644 --- a/be/src/pipeline/pipeline_fragment_context.cpp +++ b/be/src/pipeline/pipeline_fragment_context.cpp @@ -363,6 +363,7 @@ Status PipelineFragmentContext::_build_pipeline_tasks(const doris::TPipelineFrag const auto target_size = request.local_params.size(); _tasks.resize(target_size); _runtime_filter_states.resize(target_size); + _runtime_filter_mgr_map.resize(target_size); _task_runtime_states.resize(_pipelines.size()); for (size_t pip_idx = 0; pip_idx < _pipelines.size(); pip_idx++) { _task_runtime_states[pip_idx].resize(_pipelines[pip_idx]->num_tasks()); @@ -510,7 +511,7 @@ Status PipelineFragmentContext::_build_pipeline_tasks(const doris::TPipelineFrag } { std::lock_guard l(_state_map_lock); - _runtime_filter_mgr_map[fragment_instance_id] = std::move(runtime_filter_mgr); + _runtime_filter_mgr_map[i] = std::move(runtime_filter_mgr); } return Status::OK(); }; diff --git a/be/src/pipeline/pipeline_fragment_context.h b/be/src/pipeline/pipeline_fragment_context.h index d672ad6e9233c1..bd3a350d0a240a 100644 --- a/be/src/pipeline/pipeline_fragment_context.h +++ b/be/src/pipeline/pipeline_fragment_context.h @@ -275,8 +275,7 @@ class PipelineFragmentContext : public TaskExecutionContext { _op_id_to_le_state; std::map _pip_id_to_pipeline; - // UniqueId -> runtime mgr - std::map> _runtime_filter_mgr_map; + std::vector> _runtime_filter_mgr_map; //Here are two types of runtime states: // - _runtime state is at the Fragment level. diff --git a/be/src/pipeline/task_queue.h b/be/src/pipeline/task_queue.h index 1651eb50cac4ab..2218d70fde61dc 100644 --- a/be/src/pipeline/task_queue.h +++ b/be/src/pipeline/task_queue.h @@ -107,12 +107,15 @@ class MultiCoreTaskQueue { public: explicit MultiCoreTaskQueue(int core_size); +#ifndef BE_TEST ~MultiCoreTaskQueue(); - - void close(); - // Get the task by core id. PipelineTask* take(int core_id); +#else + virtual ~MultiCoreTaskQueue(); + virtual PipelineTask* take(int core_id); +#endif + void close(); // TODO combine these methods to `push_back(task, core_id = -1)` Status push_back(PipelineTask* task); diff --git a/be/src/util/jsonb_parser_simd.h b/be/src/util/jsonb_parser_simd.h index 96ce866f74e256..8684fc192188e6 100644 --- a/be/src/util/jsonb_parser_simd.h +++ b/be/src/util/jsonb_parser_simd.h @@ -136,7 +136,7 @@ class JsonbParserTSIMD { break; } case simdjson::ondemand::json_type::number: { - write_number(doc.get_number()); + write_number(doc.get_number(), doc.raw_json_token()); break; } } @@ -172,7 +172,7 @@ class JsonbParserTSIMD { break; } case simdjson::ondemand::json_type::number: { - write_number(value.get_number()); + write_number(value.get_number(), value.raw_json_token()); break; } case simdjson::ondemand::json_type::object: { @@ -290,9 +290,23 @@ class JsonbParserTSIMD { } } - void write_number(simdjson::ondemand::number num) { + void write_number(simdjson::ondemand::number num, std::string_view raw_string) { if (num.is_double()) { - if (writer_.writeDouble(num.get_double()) == 0) { + double number = num.get_double(); + // When a double exceeds the precision that can be represented by a double type in simdjson, it gets converted to 0. + // The correct approach, should be to truncate the double value instead. + if (number == 0) { + StringParser::ParseResult result; + number = StringParser::string_to_float(raw_string.data(), raw_string.size(), + &result); + if (result != StringParser::PARSE_SUCCESS) { + err_ = JsonbErrType::E_INVALID_NUMBER; + LOG(WARNING) << "invalid number, raw string is: " << raw_string; + return; + } + } + + if (writer_.writeDouble(number) == 0) { err_ = JsonbErrType::E_OUTPUT_FAIL; LOG(WARNING) << "writeDouble failed"; return; diff --git a/be/src/vec/exec/format/parquet/parquet_thrift_util.h b/be/src/vec/exec/format/parquet/parquet_thrift_util.h index b767f177f4a326..1c04d748ca54f7 100644 --- a/be/src/vec/exec/format/parquet/parquet_thrift_util.h +++ b/be/src/vec/exec/format/parquet/parquet_thrift_util.h @@ -34,7 +34,7 @@ namespace doris::vectorized { constexpr uint8_t PARQUET_VERSION_NUMBER[4] = {'P', 'A', 'R', '1'}; constexpr uint32_t PARQUET_FOOTER_SIZE = 8; -constexpr size_t INIT_META_SIZE = 128 * 1024; // 128k +constexpr size_t INIT_META_SIZE = 48 * 1024; // 48k static Status parse_thrift_footer(io::FileReaderSPtr file, FileMetaData** file_metadata, size_t* meta_size, io::IOContext* io_ctx) { diff --git a/be/test/pipeline/dummy_task_queue.h b/be/test/pipeline/dummy_task_queue.h new file mode 100644 index 00000000000000..bace0ecae96192 --- /dev/null +++ b/be/test/pipeline/dummy_task_queue.h @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "pipeline/task_queue.h" + +namespace doris::pipeline { + +class DummyTaskQueue final : public MultiCoreTaskQueue { + explicit DummyTaskQueue(int core_size) : MultiCoreTaskQueue(core_size) {} + ~DummyTaskQueue() override = default; + PipelineTask* take(int core_id) override { + PipelineTask* task = nullptr; + do { + DCHECK(_prio_task_queues.size() > core_id) + << " list size: " << _prio_task_queues.size() << " core_id: " << core_id + << " _core_size: " << _core_size << " _next_core: " << _next_core.load(); + task = _prio_task_queues[core_id].try_take(false); + if (task) { + break; + } + task = _steal_take(core_id); + if (task) { + break; + } + task = _prio_task_queues[core_id].take(1); + if (task) { + break; + } + } while (false); + if (task) { + task->pop_out_runnable_queue(); + } + return task; + } +}; +} // namespace doris::pipeline diff --git a/be/test/pipeline/pipeline_test.cpp b/be/test/pipeline/pipeline_test.cpp index d1c0af85f58d24..6466d6c2927e10 100644 --- a/be/test/pipeline/pipeline_test.cpp +++ b/be/test/pipeline/pipeline_test.cpp @@ -22,12 +22,15 @@ #include "common/exception.h" #include "common/status.h" +#include "dummy_task_queue.h" +#include "exprs/bloom_filter_func.h" +#include "exprs/hybrid_set.h" +#include "exprs/runtime_filter.h" #include "pipeline/dependency.h" #include "pipeline/exec/exchange_source_operator.h" #include "pipeline/exec/hashjoin_build_sink.h" #include "pipeline/exec/hashjoin_probe_operator.h" #include "pipeline/pipeline_fragment_context.h" -#include "pipeline/task_queue.h" #include "runtime/exec_env.h" #include "runtime/fragment_mgr.h" #include "thrift_builder.h" @@ -45,10 +48,13 @@ class PipelineTest : public testing::Test { public: PipelineTest() : _obj_pool(new ObjectPool()), - _mgr(std::make_unique()) { + _mgr(std::make_unique()) {} + ~PipelineTest() override = default; + void SetUp() override { _query_options = TQueryOptionsBuilder() .set_enable_local_exchange(true) .set_enable_local_shuffle(true) + .set_runtime_filter_max_in_num(15) .build(); auto fe_address = TNetworkAddress(); fe_address.hostname = LOCALHOST; @@ -56,10 +62,12 @@ class PipelineTest : public testing::Test { _query_ctx = QueryContext::create_shared(_query_id, ExecEnv::GetInstance(), _query_options, fe_address, true, fe_address, QuerySource::INTERNAL_FRONTEND); + _query_ctx->runtime_filter_mgr()->set_runtime_filter_params( + TRuntimeFilterParamsBuilder().build()); ExecEnv::GetInstance()->set_stream_mgr(_mgr.get()); - _task_queue = std::make_unique(1); + _task_queue = std::make_unique(1); } - ~PipelineTest() override = default; + void TearDown() override {} private: std::shared_ptr _build_pipeline(int num_instances, Pipeline* parent = nullptr) { @@ -111,6 +119,7 @@ class PipelineTest : public testing::Test { _pipeline_profiles.clear(); _pipeline_tasks.clear(); _runtime_states.clear(); + _runtime_filter_mgrs.clear(); } int _next_fragment_id() { return _fragment_id++; } int _next_node_id() { return _next_node_idx++; } @@ -125,7 +134,7 @@ class PipelineTest : public testing::Test { std::shared_ptr _query_ctx; TUniqueId _query_id = TUniqueId(); TQueryOptions _query_options; - std::unique_ptr _task_queue; + std::unique_ptr _task_queue; // Fragment level // Fragment0 -> Fragment1 @@ -149,6 +158,9 @@ class PipelineTest : public testing::Test { std::vector>> _pipeline_tasks; std::vector>> _runtime_states; + // Instance level + std::vector> _runtime_filter_mgrs; + const std::string LOCALHOST = BackendOptions::get_localhost(); const int DUMMY_PORT = config::brpc_port; }; @@ -184,7 +196,7 @@ TEST_F(PipelineTest, HAPPY_PATH) { .set_scalar_type(TPrimitiveType::INT) .build()) .build()) - .set_nullIndicatorBit(0) + .set_nullIndicatorBit(-1) .set_byteOffset(0) .set_slotIdx(0) .set_isMaterialized(true) @@ -489,7 +501,7 @@ TEST_F(PipelineTest, PLAN_LOCAL_EXCHANGE) { .set_scalar_type(TPrimitiveType::INT) .build()) .build()) - .set_nullIndicatorBit(0) + .set_nullIndicatorBit(-1) .set_byteOffset(0) .set_slotIdx(0) .set_isMaterialized(true) @@ -585,7 +597,7 @@ TEST_F(PipelineTest, PLAN_HASH_JOIN) { .set_scalar_type(TPrimitiveType::INT) .build()) .build()) - .set_nullIndicatorBit(0) + .set_nullIndicatorBit(-1) .set_byteOffset(0) .set_slotIdx(0) .set_isMaterialized(true) @@ -604,7 +616,7 @@ TEST_F(PipelineTest, PLAN_HASH_JOIN) { .set_scalar_type(TPrimitiveType::INT) .build()) .build()) - .set_nullIndicatorBit(0) + .set_nullIndicatorBit(-1) .set_byteOffset(0) .set_slotIdx(0) .set_isMaterialized(true) @@ -623,7 +635,7 @@ TEST_F(PipelineTest, PLAN_HASH_JOIN) { .set_scalar_type(TPrimitiveType::INT) .build()) .build()) - .set_nullIndicatorBit(0) + .set_nullIndicatorBit(-1) .set_byteOffset(0) .set_slotIdx(0) .set_isMaterialized(true) @@ -640,7 +652,7 @@ TEST_F(PipelineTest, PLAN_HASH_JOIN) { .set_scalar_type(TPrimitiveType::INT) .build()) .build()) - .set_nullIndicatorBit(0) + .set_nullIndicatorBit(-1) .set_byteOffset(4) .set_slotIdx(1) .set_isMaterialized(true) @@ -720,6 +732,73 @@ TEST_F(PipelineTest, PLAN_HASH_JOIN) { .append_vintermediate_tuple_id_list(1) .build()) .append_row_tuples(2, false) + .append_projections( + TExprBuilder() + .append_nodes( + TExprNodeBuilder( + TExprNodeType::SLOT_REF, + TTypeDescBuilder() + .set_types( + TTypeNodeBuilder() + .set_type( + TTypeNodeType:: + SCALAR) + .set_scalar_type( + TPrimitiveType:: + INT) + .build()) + .build(), + 0) + .set_slot_ref(TSlotRefBuilder(0, 0).build()) + .build()) + .build()) + .append_projections( + TExprBuilder() + .append_nodes( + TExprNodeBuilder( + TExprNodeType::SLOT_REF, + TTypeDescBuilder() + .set_types( + TTypeNodeBuilder() + .set_type( + TTypeNodeType:: + SCALAR) + .set_scalar_type( + TPrimitiveType:: + INT) + .build()) + .build(), + 0) + .set_slot_ref(TSlotRefBuilder(1, 1).build()) + .build()) + .build()) + .append_runtime_filters( + TRuntimeFilterDescBuilder( + 0, + TExprBuilder() + .append_nodes( + TExprNodeBuilder( + TExprNodeType::SLOT_REF, + TTypeDescBuilder() + .set_types( + TTypeNodeBuilder() + .set_type( + TTypeNodeType:: + SCALAR) + .set_scalar_type( + TPrimitiveType:: + INT) + .build()) + .build(), + 0) + .set_slot_ref( + TSlotRefBuilder(1, 1).build()) + .build()) + .build(), + 0, std::map {}) + .set_bloom_filter_size_bytes(1048576) + .set_build_bf_exactly(false) + .build()) .build(); { @@ -850,6 +929,12 @@ TEST_F(PipelineTest, PLAN_HASH_JOIN) { { // Build pipeline task int task_id = 0; + _runtime_filter_mgrs.resize(parallelism); + for (int j = 0; j < parallelism; j++) { + auto runtime_filter_state = RuntimeFilterParamsContext::create(_query_ctx.get()); + _runtime_filter_mgrs[j] = std::make_unique( + _query_id, runtime_filter_state, _query_ctx->query_mem_tracker, false); + } for (size_t i = 0; i < _pipelines.size(); i++) { EXPECT_EQ(_pipelines[i]->id(), i); _pipeline_profiles[_pipelines[i]->id()] = std::make_shared( @@ -871,6 +956,8 @@ TEST_F(PipelineTest, PLAN_HASH_JOIN) { local_runtime_state->set_task_num(_pipelines[i]->num_tasks()); local_runtime_state->set_task_execution_context( std::static_pointer_cast(_context.back())); + local_runtime_state->set_runtime_filter_mgr(_runtime_filter_mgrs[j].get()); + _runtime_filter_mgrs[j]->_state->set_state(local_runtime_state.get()); std::map, std::shared_ptr>> le_state_map; @@ -891,6 +978,7 @@ TEST_F(PipelineTest, PLAN_HASH_JOIN) { } std::shared_ptr downstream_recvr; + auto downstream_pipeline_profile = std::make_shared("Downstream Pipeline"); { // Build downstream recvr auto context = _build_fragment_context(); @@ -900,13 +988,12 @@ TEST_F(PipelineTest, PLAN_HASH_JOIN) { downstream_runtime_state->set_task_execution_context( std::static_pointer_cast(context)); - auto downstream_pipeline_profile = std::make_shared("Downstream Pipeline"); auto* memory_used_counter = downstream_pipeline_profile->AddHighWaterMarkCounter( "MemoryUsage", TUnit::BYTES, "", 1); downstream_recvr = ExecEnv::GetInstance()->_vstream_mgr->create_recvr( downstream_runtime_state.get(), memory_used_counter, _pipelines.front()->operators().back()->row_desc(), dest_ins_id, dest_node_id, - parallelism, downstream_pipeline_profile.get(), false, 20480); + parallelism, downstream_pipeline_profile.get(), false, 2048000); } for (size_t i = 0; i < _pipelines.size(); i++) { for (int j = 0; j < parallelism; j++) { @@ -914,21 +1001,16 @@ TEST_F(PipelineTest, PLAN_HASH_JOIN) { EXPECT_EQ(_pipeline_tasks[_pipelines[i]->id()][j]->prepare(scan_ranges, j, tsink, _query_ctx.get()), Status::OK()); + if (i == 1) { + auto& local_state = _runtime_states[i][j] + ->get_sink_local_state() + ->cast(); + EXPECT_EQ(local_state._runtime_filters.size(), 1); + EXPECT_EQ(local_state._should_build_hash_table, true); + } } } - // Construct input block - vectorized::Block block; - { - vectorized::DataTypePtr int_type = std::make_shared(); - - auto int_col0 = vectorized::ColumnInt32::create(); - int_col0->insert_many_vals(1, 10); - block.insert({std::move(int_col0), int_type, "test_int_col0"}); - } - auto block_mem_usage = block.allocated_bytes(); - EXPECT_GT(block_mem_usage - 1, 0); - { for (size_t i = 0; i < _pipelines.size(); i++) { for (int j = 0; j < parallelism; j++) { @@ -960,24 +1042,146 @@ TEST_F(PipelineTest, PLAN_HASH_JOIN) { { for (int i = _pipelines.size() - 1; i >= 0; i--) { for (int j = 0; j < parallelism; j++) { + bool eos = false; + EXPECT_EQ(_pipeline_tasks[i][j]->execute(&eos), Status::OK()); + EXPECT_EQ(_pipeline_tasks[i][j]->_opened, true); + EXPECT_EQ(eos, false); + } + } + } + for (int i = _pipelines.size() - 1; i >= 0; i--) { + for (int j = 0; j < parallelism; j++) { + { + vectorized::Block block; + { + vectorized::DataTypePtr int_type = + std::make_shared(); + + auto int_col0 = vectorized::ColumnInt32::create(); + if (j == 0 || i == 0) { + int_col0->insert_many_vals(j, 10); + } else { + size_t ndv = 16; + for (size_t n = 0; n < ndv; n++) { + int_col0->insert_many_vals(n, 1); + } + } + + block.insert({std::move(int_col0), int_type, "test_int_col0"}); + } auto& local_state = _runtime_states[i][j] ->get_local_state(_pipelines[i]->operators().front()->operator_id()) ->cast(); - local_state.stream_recvr->_sender_queues[0]->decrement_senders(0); - - bool eos = false; - EXPECT_EQ(_pipeline_tasks[i][j]->execute(&eos), Status::OK()); - EXPECT_EQ(_pipeline_tasks[i][j]->_is_blocked(), false); - EXPECT_EQ(eos, true); - EXPECT_EQ(_pipeline_tasks[i][j]->is_pending_finish(), false); - EXPECT_EQ(_pipeline_tasks[i][j]->close(Status::OK()), Status::OK()); + EXPECT_EQ(local_state.stream_recvr->_sender_queues[0]->_source_dependency->ready(), + false); + EXPECT_EQ(local_state.stream_recvr->_sender_queues[0] + ->_source_dependency->_blocked_task.size(), + i == 1 ? 1 : 0); + local_state.stream_recvr->_sender_queues[0]->add_block(&block, true); + } + } + } + { + // Pipeline 1 is blocked by exchange dependency so tasks are ready after data reached. + // Pipeline 0 is blocked by hash join dependency and is still waiting for upstream tasks done. + for (int j = 0; j < parallelism; j++) { + // Task is ready and be push into runnable task queue. + EXPECT_EQ(_task_queue->take(0) != nullptr, true); + } + EXPECT_EQ(_task_queue->take(0), nullptr); + for (int j = 0; j < parallelism; j++) { + EXPECT_EQ(_pipeline_tasks[1][j]->_is_blocked(), false); + } + } + { + // Pipeline 1 ran first and build hash table in join build operator. + for (int j = 0; j < parallelism; j++) { + bool eos = false; + EXPECT_EQ(_pipeline_tasks[1][j]->execute(&eos), Status::OK()); + EXPECT_EQ(eos, false); + } + for (int j = 0; j < parallelism; j++) { + auto& local_state = + _runtime_states[1][j] + ->get_local_state(_pipelines[1]->operators().front()->operator_id()) + ->cast(); + local_state.stream_recvr->_sender_queues[0]->decrement_senders(0); + + bool eos = false; + EXPECT_EQ(_pipeline_tasks[1][j]->execute(&eos), Status::OK()); + EXPECT_EQ(_pipeline_tasks[1][j]->_is_blocked(), false); + EXPECT_EQ(eos, true); + auto& sink_local_state = _runtime_states[1][j] + ->get_sink_local_state() + ->cast(); + EXPECT_EQ(sink_local_state._runtime_filters_disabled, false); + EXPECT_EQ(sink_local_state._runtime_filter_slots->_runtime_filters.size(), 1); + EXPECT_EQ(sink_local_state._runtime_filter_slots->_runtime_filters[0] + ->need_sync_filter_size(), + false); + EXPECT_EQ(sink_local_state._runtime_filter_slots->_runtime_filters[0] + ->_runtime_filter_type, + RuntimeFilterType::IN_OR_BLOOM_FILTER); + EXPECT_EQ(_pipeline_tasks[1][j]->is_pending_finish(), false); + EXPECT_EQ(_pipeline_tasks[1][j]->close(Status::OK()), Status::OK()); + EXPECT_EQ(sink_local_state._runtime_filter_slots->_runtime_filters[0]->get_real_type(), + j == 0 ? RuntimeFilterType::IN_FILTER : RuntimeFilterType::BLOOM_FILTER) + << " " << j << " " + << IRuntimeFilter::to_string( + sink_local_state._runtime_filter_slots->_runtime_filters[0] + ->get_real_type()); + EXPECT_EQ(sink_local_state._runtime_filter_slots->_runtime_filters[0] + ->_wrapper->is_ignored(), + false); + if (j == 0) { + EXPECT_EQ(sink_local_state._runtime_filter_slots->_runtime_filters[0] + ->_wrapper->_context->hybrid_set->size(), + 1); + } else { + EXPECT_EQ(sink_local_state._runtime_filter_slots->_runtime_filters[0] + ->_wrapper->_context->bloom_filter_func->_build_bf_exactly, + false); + + EXPECT_EQ(sink_local_state._runtime_filter_slots->_runtime_filters[0] + ->_wrapper->_context->bloom_filter_func->_bloom_filter_length, + 1048576); } } - { - EXPECT_EQ(downstream_recvr->_sender_queues[0]->_block_queue.size(), 0); - EXPECT_EQ(downstream_recvr->_sender_queues[0]->_num_remaining_senders, 0); + } + { + // Pipeline 0 ran once hash table is built. + for (int j = 0; j < parallelism; j++) { + EXPECT_EQ(_pipeline_tasks[0][j]->_is_blocked(), false); } + for (int j = 0; j < parallelism; j++) { + bool eos = false; + EXPECT_EQ(_pipeline_tasks[0][j]->execute(&eos), Status::OK()); + EXPECT_EQ(eos, false); + } + for (int j = 0; j < parallelism; j++) { + auto& local_state = + _runtime_states[0][j] + ->get_local_state(_pipelines[0]->operators().front()->operator_id()) + ->cast(); + local_state.stream_recvr->_sender_queues[0]->decrement_senders(0); + + bool eos = false; + EXPECT_EQ(_pipeline_tasks[0][j]->execute(&eos), Status::OK()); + EXPECT_EQ(_pipeline_tasks[0][j]->_is_blocked(), false); + EXPECT_EQ(eos, true); + EXPECT_EQ(_pipeline_tasks[0][j]->is_pending_finish(), false); + EXPECT_EQ(_pipeline_tasks[0][j]->close(Status::OK()), Status::OK()); + } + } + { + // [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] join [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] produces 100 rows in instance 0. + // [2, 2, 2, 2, 2, 2, 2, 2, 2, 2] join [2, 2, 2, 2, 2, 2, 2, 2, 2, 2] produces 100 rows in instance 1. + EXPECT_EQ(downstream_recvr->_sender_queues[0]->_block_queue.size(), 2); + EXPECT_EQ(downstream_recvr->_sender_queues[0]->_block_queue.front()._block->rows(), + 10 * 10); + EXPECT_EQ(downstream_recvr->_sender_queues[0]->_block_queue.back()._block->rows(), 10); + EXPECT_EQ(downstream_recvr->_sender_queues[0]->_num_remaining_senders, 0); } downstream_recvr->close(); } diff --git a/be/test/pipeline/thrift_builder.h b/be/test/pipeline/thrift_builder.h index bdcead8861690e..1af1ca760bd8c9 100644 --- a/be/test/pipeline/thrift_builder.h +++ b/be/test/pipeline/thrift_builder.h @@ -58,7 +58,7 @@ class TQueryOptionsBuilder { return *this; } TQueryOptionsBuilder& set_enable_new_shuffle_hash_method(bool enable_new_shuffle_hash_method) { - _query_options.enable_new_shuffle_hash_method = enable_new_shuffle_hash_method; + _query_options.__set_enable_new_shuffle_hash_method(enable_new_shuffle_hash_method); return *this; } TQueryOptionsBuilder& set_enable_local_shuffle(bool enable_local_shuffle) { @@ -66,11 +66,23 @@ class TQueryOptionsBuilder { return *this; } TQueryOptionsBuilder& set_runtime_filter_wait_infinitely(bool runtime_filter_wait_infinitely) { - _query_options.runtime_filter_wait_infinitely = runtime_filter_wait_infinitely; + _query_options.__set_runtime_filter_wait_infinitely(runtime_filter_wait_infinitely); return *this; } TQueryOptionsBuilder& set_enable_local_merge_sort(bool enable_local_merge_sort) { - _query_options.enable_local_merge_sort = enable_local_merge_sort; + _query_options.__set_enable_local_merge_sort(enable_local_merge_sort); + return *this; + } + TQueryOptionsBuilder& set_runtime_filter_max_in_num(int64_t runtime_filter_max_in_num) { + _query_options.__set_runtime_filter_max_in_num(runtime_filter_max_in_num); + return *this; + } + TQueryOptionsBuilder& set_runtime_bloom_filter_min_size(int64_t runtime_bloom_filter_min_size) { + _query_options.__set_runtime_bloom_filter_min_size(runtime_bloom_filter_min_size); + return *this; + } + TQueryOptionsBuilder& set_runtime_bloom_filter_max_size(int64_t runtime_bloom_filter_max_size) { + _query_options.__set_runtime_bloom_filter_max_size(runtime_bloom_filter_max_size); return *this; } @@ -117,6 +129,16 @@ class TPlanNodeBuilder { _plan_node.__set_output_tuple_id(output_tuple_id); return *this; } + TPlanNodeBuilder& append_projections(TExpr& projections) { + _plan_node.__isset.projections = true; + _plan_node.projections.push_back(projections); + return *this; + } + TPlanNodeBuilder& append_runtime_filters(TRuntimeFilterDesc& runtime_filter) { + _plan_node.__isset.runtime_filters = true; + _plan_node.runtime_filters.push_back(runtime_filter); + return *this; + } TPlanNode& build() { return _plan_node; } @@ -127,6 +149,55 @@ class TPlanNodeBuilder { TPlanNode _plan_node; }; +class TRuntimeFilterDescBuilder { +public: + explicit TRuntimeFilterDescBuilder( + int filter_id, TExpr& src_expr, int expr_order, + std::map planId_to_target_expr, bool is_broadcast_join = false, + bool has_local_targets = true, bool has_remote_targets = false, + TRuntimeFilterType::type type = TRuntimeFilterType::IN_OR_BLOOM) + : _desc() { + _desc.__set_filter_id(filter_id); + _desc.__set_src_expr(src_expr); + _desc.__set_expr_order(expr_order); + _desc.__set_planId_to_target_expr(planId_to_target_expr); + _desc.__set_is_broadcast_join(is_broadcast_join); + _desc.__set_has_local_targets(has_local_targets); + _desc.__set_has_remote_targets(has_remote_targets); + _desc.__set_type(type); + } + explicit TRuntimeFilterDescBuilder( + int filter_id, TExpr&& src_expr, int expr_order, + std::map planId_to_target_expr, bool is_broadcast_join = false, + bool has_local_targets = true, bool has_remote_targets = false, + TRuntimeFilterType::type type = TRuntimeFilterType::IN_OR_BLOOM) + : _desc() { + _desc.__set_filter_id(filter_id); + _desc.__set_src_expr(src_expr); + _desc.__set_expr_order(expr_order); + _desc.__set_planId_to_target_expr(planId_to_target_expr); + _desc.__set_is_broadcast_join(is_broadcast_join); + _desc.__set_has_local_targets(has_local_targets); + _desc.__set_has_remote_targets(has_remote_targets); + _desc.__set_type(type); + } + + TRuntimeFilterDescBuilder& set_bloom_filter_size_bytes(int64_t bloom_filter_size_bytes) { + _desc.__set_bloom_filter_size_bytes(bloom_filter_size_bytes); + return *this; + } + TRuntimeFilterDescBuilder& set_build_bf_exactly(bool build_bf_exactly) { + _desc.__set_build_bf_exactly(build_bf_exactly); + return *this; + } + TRuntimeFilterDesc& build() { return _desc; } + TRuntimeFilterDescBuilder(const TRuntimeFilterDescBuilder&) = delete; + void operator=(const TRuntimeFilterDescBuilder&) = delete; + +private: + TRuntimeFilterDesc _desc; +}; + class TExchangeNodeBuilder { public: explicit TExchangeNodeBuilder() : _plan_node() {} @@ -269,7 +340,10 @@ class TSlotDescriptorBuilder { class TTypeDescBuilder { public: - explicit TTypeDescBuilder() : _desc() {} + explicit TTypeDescBuilder() : _desc() { + _desc.__set_result_is_nullable(false); + _desc.__set_is_nullable(false); + } TTypeDescBuilder& set_types(TTypeNode type_node) { _desc.types.push_back(type_node); @@ -423,6 +497,7 @@ class TExprNodeBuilder { _expr_node.__set_type(type); _expr_node.__set_num_children(num_children); _expr_node.__set_opcode(opcode); + _expr_node.__set_is_nullable(false); } explicit TExprNodeBuilder(TExprNodeType::type node_type, TTypeDesc&& type, int num_children, TExprOpcode::type opcode = TExprOpcode::INVALID_OPCODE) @@ -477,4 +552,27 @@ class TEqJoinConditionBuilder { TEqJoinCondition _eq_conjuncts; }; +class TRuntimeFilterParamsBuilder { +public: + explicit TRuntimeFilterParamsBuilder( + TNetworkAddress runtime_filter_merge_addr = TNetworkAddress(), + std::map> rid_to_target_param = {}, + std::map rid_to_runtime_filter = {}, + std::map runtime_filter_builder_num = {}, + std::map> rid_to_target_paramv2 = {}) + : _params() { + _params.__set_runtime_filter_merge_addr(runtime_filter_merge_addr); + _params.__set_rid_to_target_param(rid_to_target_param); + _params.__set_rid_to_runtime_filter(rid_to_runtime_filter); + _params.__set_runtime_filter_builder_num(runtime_filter_builder_num); + _params.__set_rid_to_target_paramv2(rid_to_target_paramv2); + } + TRuntimeFilterParams& build() { return _params; } + TRuntimeFilterParamsBuilder(const TRuntimeFilterParamsBuilder&) = delete; + void operator=(const TRuntimeFilterParamsBuilder&) = delete; + +private: + TRuntimeFilterParams _params; +}; + } // namespace doris::pipeline diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h index 78d3358b581ed9..1e58f0d4d74037 100644 --- a/cloud/src/common/config.h +++ b/cloud/src/common/config.h @@ -229,6 +229,15 @@ CONF_mInt64(max_s3_client_retry, "10"); // Max byte getting delete bitmap can return, default is 1GB CONF_mInt64(max_get_delete_bitmap_byte, "1073741824"); +// Max byte txn commit when updating delete bitmap, default is 7MB. +// Because the size of one fdb transaction can't exceed 10MB, and +// fdb does not have an accurate way to estimate the size of txn. +// In my test, when txn->approximate_bytes() bigger than 8MB, +// it may meet Transaction exceeds byte limit error. We'd better +// reserve 1MB of buffer, so setting the default value to 7MB is +// more reasonable. +CONF_mInt64(max_txn_commit_byte, "7340032"); + CONF_Bool(enable_cloud_txn_lazy_commit, "true"); CONF_Int32(txn_lazy_commit_rowsets_thresold, "1000"); CONF_Int32(txn_lazy_commit_num_threads, "8"); diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index 17154a24777905..7914bf5db11cf6 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -1851,25 +1851,40 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont } // 4. Update delete bitmap for curent txn - size_t total_key = 0; - size_t total_size = 0; + size_t current_key_count = 0; + size_t current_value_count = 0; + size_t total_key_count = 0; + size_t total_value_count = 0; + size_t total_txn_put_keys = 0; + size_t total_txn_put_bytes = 0; + size_t total_txn_size = 0; for (size_t i = 0; i < request->rowset_ids_size(); ++i) { auto& key = delete_bitmap_keys.delete_bitmap_keys(i); auto& val = request->segment_delete_bitmaps(i); // Split into multiple fdb transactions, because the size of one fdb // transaction can't exceed 10MB. - if (fdb_txn_size + key.size() + val.size() > 9 * 1024 * 1024) { - LOG(INFO) << "fdb txn size more than 9MB, current size: " << fdb_txn_size - << " lock_id=" << request->lock_id(); + if (txn->approximate_bytes() + key.size() * 3 + val.size() > config::max_txn_commit_byte) { + LOG(INFO) << "fdb txn size more than " << config::max_txn_commit_byte + << ", current size: " << txn->approximate_bytes() + << " lock_id=" << request->lock_id() << ", need to commit"; err = txn->commit(); + total_txn_put_keys += txn->num_put_keys(); + total_txn_put_bytes += txn->put_bytes(); + total_txn_size += txn->approximate_bytes(); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); - ss << "failed to update delete bitmap, err=" << err; + ss << "failed to update delete bitmap, err=" << err << " tablet_id=" << tablet_id + << " lock_id=" << request->lock_id() + << " delete_bitmap_key=" << current_key_count + << " delete_bitmap_value=" << current_value_count + << " put_size=" << txn->put_bytes() << " num_put_keys=" << txn->num_put_keys() + << " txn_size=" << txn->approximate_bytes(); msg = ss.str(); return; } - fdb_txn_size = 0; + current_key_count = 0; + current_value_count = 0; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -1888,24 +1903,34 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont } // splitting large values (>90*1000) into multiple KVs cloud::put(txn.get(), key, val, 0); - fdb_txn_size = fdb_txn_size + key.size() + val.size(); - total_key++; - total_size += key.size() + val.size(); + current_key_count++; + current_value_count += val.size(); + total_key_count++; + total_value_count += val.size(); VLOG_DEBUG << "xxx update delete bitmap put delete_bitmap_key=" << hex(key) << " lock_id=" << request->lock_id() << " key_size: " << key.size() << " value_size: " << val.size(); } - err = txn->commit(); + total_txn_put_keys += txn->num_put_keys(); + total_txn_put_bytes += txn->put_bytes(); + total_txn_size += txn->approximate_bytes(); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); - ss << "failed to update delete bitmap, err=" << err; + ss << "failed to update delete bitmap, err=" << err << " tablet_id=" << tablet_id + << " lock_id=" << request->lock_id() << " delete_bitmap_key=" << current_key_count + << " delete_bitmap_value=" << current_value_count << " put_size=" << txn->put_bytes() + << " num_put_keys=" << txn->num_put_keys() << " txn_size=" << txn->approximate_bytes(); msg = ss.str(); return; } LOG(INFO) << "update_delete_bitmap tablet_id=" << tablet_id << " lock_id=" << request->lock_id() - << " rowset_num=" << request->rowset_ids_size() << " total_key=" << total_key - << " total_size=" << total_size << " unlock=" << unlock; + << " rowset_num=" << request->rowset_ids_size() + << " total_key_count=" << total_key_count + << " total_value_count=" << total_value_count << " unlock=" << unlock + << " total_txn_put_keys=" << total_txn_put_keys + << " total_txn_put_bytes=" << total_txn_put_bytes + << " total_txn_size=" << total_txn_size; } void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* controller, diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp index 307528011ea88c..62a161e3edbe52 100644 --- a/cloud/src/recycler/recycler.cpp +++ b/cloud/src/recycler/recycler.cpp @@ -24,12 +24,14 @@ #include #include +#include #include #include #include #include #include "common/stopwatch.h" +#include "meta-service/meta_service.h" #include "meta-service/meta_service_schema.h" #include "meta-service/txn_kv.h" #include "meta-service/txn_kv_error.h" @@ -249,8 +251,9 @@ void Recycler::recycle_callback() { auto instance_recycler = std::make_shared( txn_kv_, instance, _thread_pool_group, txn_lazy_committer_); - if (instance_recycler->init() != 0) { - LOG(WARNING) << "failed to init instance recycler, instance_id=" << instance_id; + if (int r = instance_recycler->init(); r != 0) { + LOG(WARNING) << "failed to init instance recycler, instance_id=" << instance_id + << " ret=" << r; continue; } std::string recycle_job_key; @@ -258,6 +261,8 @@ void Recycler::recycle_callback() { int ret = prepare_instance_recycle_job(txn_kv_.get(), recycle_job_key, instance_id, ip_port_, config::recycle_interval_seconds * 1000); if (ret != 0) { // Prepare failed + LOG(WARNING) << "failed to prepare recycle_job, instance_id=" << instance_id + << " ret=" << ret; continue; } else { std::lock_guard lock(mtx_); @@ -276,7 +281,12 @@ void Recycler::recycle_callback() { std::lock_guard lock(mtx_); recycling_instance_map_.erase(instance_id); } - LOG_INFO("finish recycle instance").tag("instance_id", instance_id); + auto elpased_ms = + ctime_ms - + duration_cast(system_clock::now().time_since_epoch()).count(); + LOG_INFO("finish recycle instance") + .tag("instance_id", instance_id) + .tag("cost_ms", elpased_ms); } } @@ -523,35 +533,37 @@ int InstanceRecycler::init_storage_vault_accessors() { LOG(WARNING) << "malformed storage vault, unable to deserialize key=" << hex(k); return -1; } + TEST_SYNC_POINT_CALLBACK("InstanceRecycler::init_storage_vault_accessors.mock_vault", + &accessor_map_, &vault); if (vault.has_hdfs_info()) { auto accessor = std::make_shared(vault.hdfs_info()); int ret = accessor->init(); if (ret != 0) { LOG(WARNING) << "failed to init hdfs accessor. instance_id=" << instance_id_ - << " resource_id=" << vault.id() << " name=" << vault.name(); - return ret; + << " resource_id=" << vault.id() << " name=" << vault.name() + << " hdfs_vault=" << vault.hdfs_info().ShortDebugString(); + continue; } accessor_map_.emplace(vault.id(), std::move(accessor)); } else if (vault.has_obj_info()) { -#ifdef UNIT_TEST - auto accessor = std::make_shared(); -#else auto s3_conf = S3Conf::from_obj_store_info(vault.obj_info()); if (!s3_conf) { - LOG(WARNING) << "failed to init object accessor, instance_id=" << instance_id_; - return -1; + LOG(WARNING) << "failed to init object accessor, invalid conf, instance_id=" + << instance_id_ << " s3_vault=" << vault.obj_info().ShortDebugString(); + continue; } std::shared_ptr accessor; int ret = S3Accessor::create(std::move(*s3_conf), &accessor); if (ret != 0) { LOG(WARNING) << "failed to init s3 accessor. instance_id=" << instance_id_ - << " resource_id=" << vault.id() << " name=" << vault.name(); - return ret; + << " resource_id=" << vault.id() << " name=" << vault.name() + << " ret=" << ret + << " s3_vault=" << vault.obj_info().ShortDebugString(); + continue; } -#endif accessor_map_.emplace(vault.id(), std::move(accessor)); } @@ -562,6 +574,13 @@ int InstanceRecycler::init_storage_vault_accessors() { return -1; } + if (accessor_map_.empty()) { + LOG(WARNING) << "no accessors for instance=" << instance_id_; + return -2; + } + LOG_INFO("finish init instance recycler number_accessors={} instance=", accessor_map_.size(), + instance_id_); + return 0; } @@ -1461,7 +1480,8 @@ int InstanceRecycler::delete_rowset_data(const std::vector int { + DCHECK(accessor_map_.count(*rid)) + << "uninitilized accessor, instance_id=" << instance_id_ + << " resource_id=" << resource_id << " path[0]=" << (*paths)[0]; + if (!accessor_map_.contains(*rid)) { + LOG_WARNING("delete rowset data accessor_map_ does not contains resouce id") + .tag("resource_id", resource_id) + .tag("instance_id", instance_id_); + return -1; + } auto& accessor = accessor_map_[*rid]; - DCHECK(accessor); return accessor->delete_files(*paths); }); } @@ -1576,7 +1604,9 @@ int InstanceRecycler::delete_rowset_data(const std::string& resource_id, int64_t if (it == accessor_map_.end()) { LOG_WARNING("instance has no such resource id") .tag("instance_id", instance_id_) - .tag("resource_id", resource_id); + .tag("resource_id", resource_id) + .tag("tablet_id", tablet_id) + .tag("rowset_id", rowset_id); return -1; } auto& accessor = it->second; @@ -1588,42 +1618,107 @@ int InstanceRecycler::recycle_tablet(int64_t tablet_id) { .tag("instance_id", instance_id_) .tag("tablet_id", tablet_id); + int ret = 0; auto start_time = steady_clock::now(); + // collect resource ids + std::string rs_key0 = meta_rowset_key({instance_id_, tablet_id, 0}); + std::string rs_key1 = meta_rowset_key({instance_id_, tablet_id + 1, 0}); + std::string recyc_rs_key0 = recycle_rowset_key({instance_id_, tablet_id, ""}); + std::string recyc_rs_key1 = recycle_rowset_key({instance_id_, tablet_id + 1, ""}); + + std::set resource_ids; + int64_t recycle_rowsets_number = 0; + int64_t recycle_segments_number = 0; + int64_t recycle_rowsets_data_size = 0; + int64_t recycle_rowsets_index_size = 0; + int64_t max_rowset_version = 0; + int64_t min_rowset_creation_time = INT64_MAX; + int64_t max_rowset_creation_time = 0; + int64_t min_rowset_expiration_time = INT64_MAX; + int64_t max_rowset_expiration_time = 0; + std::unique_ptr> defer_log_statistics((int*)0x01, [&](int*) { auto cost = duration(steady_clock::now() - start_time).count(); LOG_INFO("recycle the rowsets of dropped tablet finished, cost={}s", cost) .tag("instance_id", instance_id_) - .tag("tablet_id", tablet_id); + .tag("tablet_id", tablet_id) + .tag("recycle rowsets number", recycle_rowsets_number) + .tag("recycle segments number", recycle_segments_number) + .tag("all rowsets recycle data size", recycle_rowsets_data_size) + .tag("all rowsets recycle index size", recycle_rowsets_index_size) + .tag("max rowset version", max_rowset_version) + .tag("min rowset creation time", min_rowset_creation_time) + .tag("max rowset creation time", max_rowset_creation_time) + .tag("min rowset expiration time", min_rowset_expiration_time) + .tag("max rowset expiration time", max_rowset_expiration_time) + .tag("ret", ret); }); - // delete all rowset kv in this tablet - std::string rs_key0 = meta_rowset_key({instance_id_, tablet_id, 0}); - std::string rs_key1 = meta_rowset_key({instance_id_, tablet_id + 1, 0}); - std::string recyc_rs_key0 = recycle_rowset_key({instance_id_, tablet_id, ""}); - std::string recyc_rs_key1 = recycle_rowset_key({instance_id_, tablet_id + 1, ""}); - - int ret = 0; std::unique_ptr txn; if (txn_kv_->create_txn(&txn) != TxnErrorCode::TXN_OK) { - LOG(WARNING) << "failed to delete rowset kv of tablet " << tablet_id; + LOG_WARNING("failed to recycle tablet ") + .tag("tablet id", tablet_id) + .tag("instance_id", instance_id_) + .tag("reason", "failed to create txn"); ret = -1; } - txn->remove(rs_key0, rs_key1); - txn->remove(recyc_rs_key0, recyc_rs_key1); - - // remove delete bitmap for MoW table - std::string pending_key = meta_pending_delete_bitmap_key({instance_id_, tablet_id}); - txn->remove(pending_key); - std::string delete_bitmap_start = meta_delete_bitmap_key({instance_id_, tablet_id, "", 0, 0}); - std::string delete_bitmap_end = meta_delete_bitmap_key({instance_id_, tablet_id + 1, "", 0, 0}); - txn->remove(delete_bitmap_start, delete_bitmap_end); - - TxnErrorCode err = txn->commit(); - if (err != TxnErrorCode::TXN_OK) { - LOG(WARNING) << "failed to delete rowset kv of tablet " << tablet_id << ", err=" << err; + GetRowsetResponse resp; + std::string msg; + MetaServiceCode code = MetaServiceCode::OK; + // get rowsets in tablet + internal_get_rowset(txn.get(), 0, std::numeric_limits::max() - 1, instance_id_, + tablet_id, code, msg, &resp); + if (code != MetaServiceCode::OK) { + LOG_WARNING("failed to get rowsets of tablet when recycle tablet") + .tag("tablet id", tablet_id) + .tag("msg", msg) + .tag("code", code) + .tag("instance id", instance_id_); ret = -1; } + TEST_SYNC_POINT_CALLBACK("InstanceRecycler::recycle_tablet.create_rowset_meta", &resp); + + for (const auto& rs_meta : resp.rowset_meta()) { + if (!rs_meta.has_resource_id()) { + LOG_WARNING("rowset meta does not have a resource id, impossible!") + .tag("rs_meta", rs_meta.ShortDebugString()) + .tag("instance_id", instance_id_) + .tag("tablet_id", tablet_id); + return -1; + } + auto it = accessor_map_.find(rs_meta.resource_id()); + // possible if the accessor is not initilized correctly + if (it == accessor_map_.end()) [[unlikely]] { + LOG_WARNING( + "failed to find resource id when recycle tablet, skip this vault accessor " + "recycle process") + .tag("tablet id", tablet_id) + .tag("instance_id", instance_id_) + .tag("resource_id", rs_meta.resource_id()) + .tag("rowset meta pb", rs_meta.ShortDebugString()); + return -1; + } + recycle_rowsets_number += 1; + recycle_segments_number += rs_meta.num_segments(); + recycle_rowsets_data_size += rs_meta.data_disk_size(); + recycle_rowsets_index_size += rs_meta.index_disk_size(); + max_rowset_version = std::max(max_rowset_version, rs_meta.end_version()); + min_rowset_creation_time = std::min(min_rowset_creation_time, rs_meta.creation_time()); + max_rowset_creation_time = std::max(max_rowset_creation_time, rs_meta.creation_time()); + min_rowset_expiration_time = std::min(min_rowset_expiration_time, rs_meta.txn_expiration()); + max_rowset_expiration_time = std::max(max_rowset_expiration_time, rs_meta.txn_expiration()); + resource_ids.emplace(rs_meta.resource_id()); + } + + LOG_INFO("recycle tablet start to delete object") + .tag("instance id", instance_id_) + .tag("tablet id", tablet_id) + .tag("recycle tablet resource ids are", + std::accumulate(resource_ids.begin(), resource_ids.end(), std::string(), + [](const std::string& a, const std::string& b) { + return a.empty() ? b : a + "," + b; + })); SyncExecutor concurrent_delete_executor( _thread_pool_group.s3_producer_pool, @@ -1631,16 +1726,20 @@ int InstanceRecycler::recycle_tablet(int64_t tablet_id) { [](const int& ret) { return ret != 0; }); // delete all rowset data in this tablet - for (auto& [_, accessor] : accessor_map_) { - concurrent_delete_executor.add([&, accessor_ptr = &accessor]() { - if ((*accessor_ptr)->delete_directory(tablet_path_prefix(tablet_id)) != 0) { + // ATTN: there may be data leak if not all accessor initilized successfully + // partial data deleted if the tablet is stored cross-storage vault + // vault id is not attached to TabletMeta... + for (const auto& resource_id : resource_ids) { + concurrent_delete_executor.add([&, accessor_ptr = accessor_map_[resource_id]]() { + if (accessor_ptr->delete_directory(tablet_path_prefix(tablet_id)) != 0) { LOG(WARNING) << "failed to delete rowset data of tablet " << tablet_id - << " s3_path=" << accessor->uri(); + << " path=" << accessor_ptr->uri(); return -1; } return 0; }); } + bool finished = true; std::vector rets = concurrent_delete_executor.when_all(&finished); for (int r : rets) { @@ -1651,6 +1750,40 @@ int InstanceRecycler::recycle_tablet(int64_t tablet_id) { ret = finished ? ret : -1; + if (ret != 0) { // failed recycle tablet data + LOG_WARNING("ret!=0") + .tag("finished", finished) + .tag("ret", ret) + .tag("instance_id", instance_id_) + .tag("tablet_id", tablet_id); + return ret; + } + + txn.reset(); + if (txn_kv_->create_txn(&txn) != TxnErrorCode::TXN_OK) { + LOG_WARNING("failed to recycle tablet ") + .tag("tablet id", tablet_id) + .tag("instance_id", instance_id_) + .tag("reason", "failed to create txn"); + ret = -1; + } + // delete all rowset kv in this tablet + txn->remove(rs_key0, rs_key1); + txn->remove(recyc_rs_key0, recyc_rs_key1); + + // remove delete bitmap for MoW table + std::string pending_key = meta_pending_delete_bitmap_key({instance_id_, tablet_id}); + txn->remove(pending_key); + std::string delete_bitmap_start = meta_delete_bitmap_key({instance_id_, tablet_id, "", 0, 0}); + std::string delete_bitmap_end = meta_delete_bitmap_key({instance_id_, tablet_id + 1, "", 0, 0}); + txn->remove(delete_bitmap_start, delete_bitmap_end); + + TxnErrorCode err = txn->commit(); + if (err != TxnErrorCode::TXN_OK) { + LOG(WARNING) << "failed to delete rowset kv of tablet " << tablet_id << ", err=" << err; + ret = -1; + } + if (ret == 0) { // All object files under tablet have been deleted std::lock_guard lock(recycled_tablets_mtx_); @@ -2233,7 +2366,7 @@ int InstanceRecycler::abort_timeout_txn() { txn_info.set_status(TxnStatusPB::TXN_STATUS_ABORTED); txn_info.set_finish_time(current_time); txn_info.set_reason("timeout"); - VLOG_DEBUG << "txn_info=" << txn_info.DebugString(); + VLOG_DEBUG << "txn_info=" << txn_info.ShortDebugString(); txn_inf_val.clear(); if (!txn_info.SerializeToString(&txn_inf_val)) { LOG_WARNING("failed to serialize txn info").tag("key", hex(k)); @@ -2911,7 +3044,7 @@ int InstanceRecycler::recycle_expired_stage_objects() { const auto& old_obj = instance_info_.obj_info()[idx - 1]; auto s3_conf = S3Conf::from_obj_store_info(old_obj); if (!s3_conf) { - LOG(WARNING) << "failed to init s3_conf with obj_info=" << old_obj.DebugString(); + LOG(WARNING) << "failed to init s3_conf with obj_info=" << old_obj.ShortDebugString(); continue; } diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp index b7004716035a46..fb17c29629b04e 100644 --- a/cloud/test/meta_service_test.cpp +++ b/cloud/test/meta_service_test.cpp @@ -104,6 +104,24 @@ std::unique_ptr get_meta_service() { return get_meta_service(true); } +std::unique_ptr get_fdb_meta_service() { + config::fdb_cluster_file_path = "fdb.cluster"; + static auto txn_kv = std::dynamic_pointer_cast(std::make_shared()); + static std::atomic init {false}; + bool tmp = false; + if (init.compare_exchange_strong(tmp, true)) { + int ret = txn_kv->init(); + [&] { + ASSERT_EQ(ret, 0); + ASSERT_NE(txn_kv.get(), nullptr); + }(); + } + auto rs = std::make_shared(txn_kv); + auto rl = std::make_shared(); + auto meta_service = std::make_unique(txn_kv, rs, rl); + return std::make_unique(std::move(meta_service)); +} + static std::string next_rowset_id() { static int cnt = 0; return std::to_string(++cnt); @@ -4857,6 +4875,43 @@ static std::string generate_random_string(int length) { return randomString; } +TEST(MetaServiceTest, UpdateDeleteBitmapWithBigKeys) { + auto meta_service = get_fdb_meta_service(); + // get delete bitmap update lock + brpc::Controller cntl; + GetDeleteBitmapUpdateLockRequest get_lock_req; + GetDeleteBitmapUpdateLockResponse get_lock_res; + get_lock_req.set_cloud_unique_id("test_cloud_unique_id"); + get_lock_req.set_table_id(1999); + get_lock_req.add_partition_ids(123); + get_lock_req.set_expiration(5); + get_lock_req.set_lock_id(-1); + get_lock_req.set_initiator(100); + meta_service->get_delete_bitmap_update_lock( + reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &get_lock_req, + &get_lock_res, nullptr); + ASSERT_EQ(get_lock_res.status().code(), MetaServiceCode::OK); + UpdateDeleteBitmapRequest update_delete_bitmap_req; + UpdateDeleteBitmapResponse update_delete_bitmap_res; + update_delete_bitmap_req.set_cloud_unique_id("test_cloud_unique_id"); + update_delete_bitmap_req.set_table_id(1999); + update_delete_bitmap_req.set_partition_id(123); + update_delete_bitmap_req.set_lock_id(-1); + update_delete_bitmap_req.set_initiator(100); + update_delete_bitmap_req.set_tablet_id(333); + std::string large_value = generate_random_string(300 * 1000 * 3); + for (int i = 0; i < 100000; i++) { + update_delete_bitmap_req.add_rowset_ids("0200000003ea308a3647dbea83220ed4b8897f2288244a91"); + update_delete_bitmap_req.add_segment_ids(0); + update_delete_bitmap_req.add_versions(i); + update_delete_bitmap_req.add_segment_delete_bitmaps("1"); + } + meta_service->update_delete_bitmap(reinterpret_cast(&cntl), + &update_delete_bitmap_req, &update_delete_bitmap_res, + nullptr); + ASSERT_EQ(update_delete_bitmap_res.status().code(), MetaServiceCode::OK); +} + TEST(MetaServiceTest, UpdateDeleteBitmap) { auto meta_service = get_meta_service(); diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp index e38d25aaa8420a..567d27f5d6f3c4 100644 --- a/cloud/test/recycler_test.cpp +++ b/cloud/test/recycler_test.cpp @@ -36,6 +36,7 @@ #include "meta-service/mem_txn_kv.h" #include "meta-service/meta_service.h" #include "meta-service/txn_kv_error.h" +#include "mock_accessor.h" #include "mock_resource_manager.h" #include "rate-limiter/rate_limiter.h" #include "recycler/checker.h" @@ -3263,4 +3264,134 @@ TEST(RecyclerTest, delete_rowset_data_without_inverted_index_storage_format) { } } +TEST(RecyclerTest, init_vault_accessor_failed_test) { + auto* sp = SyncPoint::get_instance(); + std::unique_ptr> defer((int*)0x01, [&sp](int*) { + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); + }); + + auto txn_kv = std::make_shared(); + EXPECT_EQ(txn_kv->init(), 0); + std::unique_ptr txn; + ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK); + std::string key; + std::string val; + + InstanceKeyInfo key_info {"test_instance"}; + instance_key(key_info, &key); + InstanceInfoPB instance; + instance.set_instance_id("GetObjStoreInfoTestInstance"); + // failed to init because S3Conf::from_obj_store_info() fails + { + ObjectStoreInfoPB obj_info; + StorageVaultPB vault; + obj_info.set_id("id"); + obj_info.set_ak("ak"); + obj_info.set_sk("sk"); + vault.mutable_obj_info()->MergeFrom(obj_info); + vault.set_name("test_failed_s3_vault_1"); + vault.set_id("failed_s3_1"); + instance.add_storage_vault_names(vault.name()); + instance.add_resource_ids(vault.id()); + txn->put(storage_vault_key({instance.instance_id(), "1"}), vault.SerializeAsString()); + } + + // succeed to init but unuseful + { + ObjectStoreInfoPB obj_info; + StorageVaultPB vault; + obj_info.set_id("id"); + obj_info.set_ak("ak"); + obj_info.set_sk("sk"); + obj_info.set_provider(ObjectStoreInfoPB_Provider_COS); + vault.mutable_obj_info()->MergeFrom(obj_info); + vault.set_name("test_failed_s3_vault_2"); + vault.set_id("failed_s3_2"); + instance.add_storage_vault_names(vault.name()); + instance.add_resource_ids(vault.id()); + instance.set_instance_id("GetObjStoreInfoTestInstance"); + txn->put(storage_vault_key({instance.instance_id(), "2"}), vault.SerializeAsString()); + } + + // failed to init because accessor->init() fails + { + HdfsBuildConf hdfs_build_conf; + StorageVaultPB vault; + hdfs_build_conf.set_fs_name("fs_name"); + hdfs_build_conf.set_user("root"); + HdfsVaultInfo hdfs_info; + hdfs_info.set_prefix("root_path"); + hdfs_info.mutable_build_conf()->MergeFrom(hdfs_build_conf); + vault.mutable_hdfs_info()->MergeFrom(hdfs_info); + vault.set_name("test_failed_hdfs_vault_1"); + vault.set_id("failed_hdfs_1"); + instance.add_storage_vault_names(vault.name()); + instance.add_resource_ids(vault.id()); + instance.set_instance_id("GetObjStoreInfoTestInstance"); + txn->put(storage_vault_key({instance.instance_id(), "3"}), vault.SerializeAsString()); + } + + auto accessor = std::make_shared(); + EXPECT_EQ(accessor->put_file("data/0/test.csv", ""), 0); + sp->set_call_back( + "InstanceRecycler::init_storage_vault_accessors.mock_vault", [&accessor](auto&& args) { + auto* map = try_any_cast< + std::unordered_map>*>( + args[0]); + auto* vault = try_any_cast(args[1]); + if (vault->name() == "test_success_hdfs_vault") { + map->emplace(vault->id(), accessor); + } + }); + sp->set_call_back("InstanceRecycler::recycle_tablet.create_rowset_meta", [](auto&& args) { + auto* resp = try_any_cast(args[0]); + auto* rs = resp->add_rowset_meta(); + rs->set_resource_id("failed_s3_2"); + rs = resp->add_rowset_meta(); + rs->set_resource_id("success_vault"); + }); + sp->enable_processing(); + + // succeed to init MockAccessor + { + HdfsBuildConf hdfs_build_conf; + StorageVaultPB vault; + hdfs_build_conf.set_fs_name("fs_name"); + hdfs_build_conf.set_user("root"); + HdfsVaultInfo hdfs_info; + hdfs_info.set_prefix("root_path"); + hdfs_info.mutable_build_conf()->MergeFrom(hdfs_build_conf); + vault.mutable_hdfs_info()->MergeFrom(hdfs_info); + vault.set_name("test_success_hdfs_vault"); + vault.set_id("success_vault"); + instance.add_storage_vault_names(vault.name()); + instance.add_resource_ids(vault.id()); + instance.set_instance_id("GetObjStoreInfoTestInstance"); + txn->put(storage_vault_key({instance.instance_id(), "4"}), vault.SerializeAsString()); + } + + val = instance.SerializeAsString(); + txn->put(key, val); + EXPECT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + + EXPECT_EQ(accessor->exists("data/0/test.csv"), 0); + + InstanceRecycler recycler(txn_kv, instance, thread_group, + std::make_shared(txn_kv)); + EXPECT_EQ(recycler.init(), 0); + EXPECT_EQ(recycler.accessor_map_.size(), 2); + + // unuseful obj accessor + EXPECT_EQ(recycler.accessor_map_.at("failed_s3_2")->exists("data/0/test.csv"), -1); + // useful mock accessor + EXPECT_EQ(recycler.accessor_map_.at("success_vault")->exists("data/0/test.csv"), 0); + + // recycle tablet will fail because unuseful obj accessor can not connectted + EXPECT_EQ(recycler.recycle_tablet(0), -1); + // however, useful mock accessor can recycle tablet + EXPECT_EQ(recycler.accessor_map_.at("success_vault")->exists("data/0/test.csv"), 1); +} + } // namespace doris::cloud diff --git a/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/update-location.sh b/docker/thirdparties/docker-compose/common/event-hook.sh old mode 100755 new mode 100644 similarity index 74% rename from docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/update-location.sh rename to docker/thirdparties/docker-compose/common/event-hook.sh index 8d727b2308dd05..144550e8bb0aeb --- a/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/update-location.sh +++ b/docker/thirdparties/docker-compose/common/event-hook.sh @@ -16,10 +16,10 @@ # specific language governing permissions and limitations # under the License. -/usr/bin/mysqld_safe & -while ! mysqladmin ping -proot --silent; do sleep 1; done - -hive --service metatool -updateLocation hdfs://hadoop-master-2:9000/user/hive/warehouse hdfs://hadoop-master:9000/user/hive/warehouse - -killall mysqld -while pgrep mysqld; do sleep 1; done +function exec_success_hook() { + echo "Executing success hook" + echo "Creating /tmp/success and /tmp/SUCCESS" + touch /tmp/success /tmp/SUCCESS + echo "Do not exit, just tailing /dev/null" + tail -f /dev/null +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/common/hive-configure.sh b/docker/thirdparties/docker-compose/common/hive-configure.sh new file mode 100755 index 00000000000000..cb17d4d2275554 --- /dev/null +++ b/docker/thirdparties/docker-compose/common/hive-configure.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Referenced from [docker-hive](https://github.com/big-data-europe/docker-hive) + +# Set some sensible defaults +export CORE_CONF_fs_defaultFS=${CORE_CONF_fs_defaultFS:-hdfs://`hostname -f`:8020} + +function addProperty() { + local path=$1 + local name=$2 + local value=$3 + + local entry="$name${value}" + local escapedEntry=$(echo $entry | sed 's/\//\\\//g') + sed -i "/<\/configuration>/ s/.*/${escapedEntry}\n&/" $path +} + +function configure() { + local path=$1 + local module=$2 + local envPrefix=$3 + + local var + local value + + echo "Configuring $module" + for c in `printenv | perl -sne 'print "$1 " if m/^${envPrefix}_(.+?)=.*/' -- -envPrefix=$envPrefix`; do + name=`echo ${c} | perl -pe 's/___/-/g; s/__/_/g; s/_/./g'` + var="${envPrefix}_${c}" + value=${!var} + echo " - Setting $name=$ " + addProperty $path $name "$value" + done +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf b/docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf index 36547b8f89d163..83fe29c2cb2fa0 100644 --- a/docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf +++ b/docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf @@ -32,21 +32,18 @@ [realms] LABS.TERADATA.COM = { - kdc = hadoop-master:88 - admin_server = hadoop-master + kdc = hadoop-master:5588 + admin_server = hadoop-master:5749 } OTHERLABS.TERADATA.COM = { - kdc = hadoop-master:89 - admin_server = hadoop-master + kdc = hadoop-master:5589 + admin_server = hadoop-master:5750 } - OTHERLABS.TERADATA.COM = { - kdc = hadoop-master:89 - admin_server = hadoop-master - } -OTHERREALM.COM = { - kdc = hadoop-master-2:88 - admin_server = hadoop-master + OTHERREALM.COM = { + kdc = hadoop-master-2:6688 + admin_server = hadoop-master-2:6749 } [domain_realm] hadoop-master-2 = OTHERREALM.COM + hadoop-master = LABS.TERADATA.COM diff --git a/docker/thirdparties/docker-compose/kerberos/common/hadoop/hadoop-run.sh b/docker/thirdparties/docker-compose/kerberos/common/hadoop/hadoop-run.sh index b8bfd8715e9565..93c6e385effe1f 100755 --- a/docker/thirdparties/docker-compose/kerberos/common/hadoop/hadoop-run.sh +++ b/docker/thirdparties/docker-compose/kerberos/common/hadoop/hadoop-run.sh @@ -38,5 +38,7 @@ fi trap exit INT echo "Running services with supervisord" +rm -rf /etc/supervisord.d/socks-proxy.conf +rm -rf /etc/supervisord.d/sshd.conf supervisord -c /etc/supervisord.conf diff --git a/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/kdc.conf.tpl b/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/kdc.conf.tpl new file mode 100644 index 00000000000000..e16c70e16db817 --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/kdc.conf.tpl @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[kdcdefaults] + kdc_ports = ${KDC_PORT1} + kdc_tcp_ports = ${KDC_PORT1} + kadmind_port = ${KADMIND_PORT1} + kpasswd_port = ${KPASSWD_PORT1} + +[realms] + LABS.TERADATA.COM = { + acl_file = /var/kerberos/krb5kdc/kadm5.acl + dict_file = /usr/share/dict/words + admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab + supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal + kdc_listen = ${KDC_PORT1} + kdc_tcp_listen = ${KDC_PORT1} + kdc_ports = ${KDC_PORT1} + kdc_tcp_ports = ${KDC_PORT1} + kadmind_port = ${KADMIND_PORT1} + kpasswd_port = ${KPASSWD_PORT1} + } + + OTHERLABS.TERADATA.COM = { + acl_file = /var/kerberos/krb5kdc/kadm5-other.acl + dict_file = /usr/share/dict/words + admin_keytab = /var/kerberos/krb5kdc/kadm5-other.keytab + supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal + kdc_listen = ${KDC_PORT2} + kdc_tcp_listen = ${KDC_PORT2} + kdc_ports = ${KDC_PORT2} + kdc_tcp_ports = ${KDC_PORT2} + kadmind_port = ${KADMIND_PORT2} + kpasswd_port = ${KPASSWD_PORT2} + } \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/krb5.conf.tpl b/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/krb5.conf.tpl new file mode 100644 index 00000000000000..1edf2bb8fd05fe --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/krb5.conf.tpl @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[logging] + default = FILE:/var/log/krb5libs.log + kdc = FILE:/var/log/krb5kdc.log + admin_server = FILE:/var/log/kadmind.log + +[libdefaults] + default_realm = LABS.TERADATA.COM + dns_lookup_realm = false + dns_lookup_kdc = false + forwardable = true + allow_weak_crypto = true + +[realms] + LABS.TERADATA.COM = { + kdc = ${HOST}:${KDC_PORT1} + admin_server = ${HOST}:${KADMIND_PORT1} + } + OTHERLABS.TERADATA.COM = { + kdc = ${HOST}:${KDC_PORT2} + admin_server = ${HOST}:${KADMIND_PORT2} + } \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/kerberos/conf/kerberos2/kdc.conf.tpl b/docker/thirdparties/docker-compose/kerberos/conf/kerberos2/kdc.conf.tpl new file mode 100644 index 00000000000000..61b4994ad5c1ac --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/conf/kerberos2/kdc.conf.tpl @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[kdcdefaults] + kdc_ports = ${KDC_PORT1} + kdc_tcp_ports = ${KDC_PORT1} + kadmind_port = ${KADMIND_PORT1} + kpasswd_port = ${KPASSWD_PORT1} + + +[realms] + OTHERREALM.COM = { + acl_file = /var/kerberos/krb5kdc/kadm5.acl + dict_file = /usr/share/dict/words + admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab + supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal + kdc_listen = ${KDC_PORT1} + kdc_tcp_listen = ${KDC_PORT1} + kdc_ports = ${KDC_PORT1} + kdc_tcp_ports = ${KDC_PORT1} + kadmind_port = ${KADMIND_PORT1} + kpasswd_port = ${KPASSWD_PORT1} + } \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/kerberos/conf/kerberos2/krb5.conf.tpl b/docker/thirdparties/docker-compose/kerberos/conf/kerberos2/krb5.conf.tpl new file mode 100644 index 00000000000000..c817dbdd79753d --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/conf/kerberos2/krb5.conf.tpl @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[logging] + default = FILE:/var/log/krb5libs.log + kdc = FILE:/var/log/krb5kdc.log + admin_server = FILE:/var/log/kadmind.log + +[libdefaults] + default_realm = OTHERREALM.COM + dns_lookup_realm = false + dns_lookup_kdc = false + forwardable = true + allow_weak_crypto = true + +[realms] + OTHERREALM.COM = { + kdc = ${HOST}:${KDC_PORT1} + admin_server = ${HOST}:${KADMIND_PORT1} + } \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/kerberos/ccache/create_kerberos_credential_cache_files.sh b/docker/thirdparties/docker-compose/kerberos/conf/my.cnf.tpl similarity index 55% rename from docker/thirdparties/docker-compose/kerberos/ccache/create_kerberos_credential_cache_files.sh rename to docker/thirdparties/docker-compose/kerberos/conf/my.cnf.tpl index 2bba3f928b198c..e91c65c100463f 100644 --- a/docker/thirdparties/docker-compose/kerberos/ccache/create_kerberos_credential_cache_files.sh +++ b/docker/thirdparties/docker-compose/kerberos/conf/my.cnf.tpl @@ -16,18 +16,23 @@ # specific language governing permissions and limitations # under the License. -set -exuo pipefail +[mysqld] +port=${MYSQL_PORT} +datadir=/var/lib/mysql +socket=/var/lib/mysql/mysql.sock +# Disabling symbolic-links is recommended to prevent assorted security risks +symbolic-links=0 +# Settings user and group are ignored when systemd is used. +# If you need to run mysqld under a different user or group, +# customize your systemd unit file for mariadb according to the +# instructions in http://fedoraproject.org/wiki/Systemd -TICKET_LIFETIME='30m' +[mysqld_safe] +log-error=/var/log/mariadb/mariadb.log +pid-file=/var/run/mariadb/mariadb.pid -kinit -l "$TICKET_LIFETIME" -f -c /etc/trino/conf/presto-server-krbcc \ - -kt /etc/trino/conf/presto-server.keytab presto-server/$(hostname -f)@LABS.TERADATA.COM - -kinit -l "$TICKET_LIFETIME" -f -c /etc/trino/conf/hive-presto-master-krbcc \ - -kt /etc/trino/conf/hive-presto-master.keytab hive/$(hostname -f)@LABS.TERADATA.COM - -kinit -l "$TICKET_LIFETIME" -f -c /etc/trino/conf/hdfs-krbcc \ - -kt /etc/hadoop/conf/hdfs.keytab hdfs/hadoop-master@LABS.TERADATA.COM +# +# include all files from the config directory +# +!includedir /etc/my.cnf.d -kinit -l "$TICKET_LIFETIME" -f -c /etc/trino/conf/hive-krbcc \ - -kt /etc/hive/conf/hive.keytab hive/hadoop-master@LABS.TERADATA.COM diff --git a/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master-2.sh b/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master-2.sh deleted file mode 100755 index eb95c5cb697619..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master-2.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -euo pipefail - -echo "Copying kerberos keytabs to /keytabs/" -mkdir -p /etc/hadoop-init.d/ -cp /etc/trino/conf/hive-presto-master.keytab /keytabs/other-hive-presto-master.keytab -cp /etc/trino/conf/presto-server.keytab /keytabs/other-presto-server.keytab -cp /keytabs/update-location.sh /etc/hadoop-init.d/update-location.sh -/usr/local/hadoop-run.sh & - -# check healthy hear -echo "Waiting for hadoop to be healthy" - -for i in {1..10}; do - if /usr/local/health.sh; then - echo "Hadoop is healthy" - break - fi - echo "Hadoop is not healthy yet. Retrying in 20 seconds..." - sleep 20 -done - -if [ $i -eq 10 ]; then - echo "Hadoop did not become healthy after 120 attempts. Exiting." - exit 1 -fi - -echo "Init kerberos test data" -kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master-2@OTHERREALM.COM -hive -f /usr/local/sql/create_kerberos_hive_table.sql -touch /mnt/SUCCESS - -tail -f /dev/null diff --git a/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh b/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh index 76f49724297a61..f12b76734f74dc 100755 --- a/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh +++ b/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh @@ -15,34 +15,60 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - set -euo pipefail +source /usr/local/common/hive-configure.sh +source /usr/local/common/event-hook.sh + +echo "Configuring hive" +configure /etc/hive/conf/hive-site.xml hive HIVE_SITE_CONF +configure /etc/hive/conf/hiveserver2-site.xml hive HIVE_SITE_CONF +configure /etc/hadoop/conf/core-site.xml core CORE_CONF +configure /etc/hadoop/conf/hdfs-site.xml hdfs HDFS_CONF +configure /etc/hadoop/conf/yarn-site.xml yarn YARN_CONF +configure /etc/hadoop/conf/mapred-site.xml mapred MAPRED_CONF +configure /etc/hive/conf/beeline-site.xml beeline BEELINE_SITE_CONF echo "Copying kerberos keytabs to keytabs/" mkdir -p /etc/hadoop-init.d/ -cp /etc/trino/conf/* /keytabs/ + +if [ "$1" == "1" ]; then + cp /etc/trino/conf/* /keytabs/ +elif [ "$1" == "2" ]; then + cp /etc/trino/conf/hive-presto-master.keytab /keytabs/other-hive-presto-master.keytab + cp /etc/trino/conf/presto-server.keytab /keytabs/other-presto-server.keytab +else + echo "Invalid index parameter. Exiting." + exit 1 +fi /usr/local/hadoop-run.sh & # check healthy hear echo "Waiting for hadoop to be healthy" -for i in {1..10}; do +for i in {1..60}; do if /usr/local/health.sh; then echo "Hadoop is healthy" break fi - echo "Hadoop is not healthy yet. Retrying in 20 seconds..." - sleep 20 + echo "Hadoop is not healthy yet. Retrying in 60 seconds..." + sleep 5 done -if [ $i -eq 10 ]; then - echo "Hadoop did not become healthy after 120 attempts. Exiting." +if [ $i -eq 60 ]; then + echo "Hadoop did not become healthy after 60 attempts. Exiting." exit 1 fi echo "Init kerberos test data" -kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master@LABS.TERADATA.COM + +if [ "$1" == "1" ]; then + kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master@LABS.TERADATA.COM +elif [ "$1" == "2" ]; then + kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master-2@OTHERREALM.COM +else + echo "Invalid index parameter. Exiting." + exit 1 +fi hive -f /usr/local/sql/create_kerberos_hive_table.sql -touch /mnt/SUCCESS -tail -f /dev/null +exec_success_hook \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl b/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl new file mode 100644 index 00000000000000..41c95057a92207 --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl @@ -0,0 +1,65 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:mysql://127.0.0.1:${MYSQL_PORT}/metastore +HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName=com.mysql.jdbc.Driver +HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName=root +HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword=root +HIVE_SITE_CONF_datanucleus_autoCreateSchema=false +HIVE_SITE_CONF_hive_metastore_port=${HMS_PORT} +HIVE_SITE_CONF_hive_metastore_uris=thrift://${IP_HOST}:${HMS_PORT} +HIVE_SITE_CONF_hive_server2_thrift_bind_host=0.0.0.0 +HIVE_SITE_CONF_hive_server2_thrift_port=${HS_PORT} +HIVE_SITE_CONF_hive_server2_webui_port=0 +HIVE_SITE_CONF_hive_compactor_initiator_on=true +HIVE_SITE_CONF_hive_compactor_worker_threads=2 +HIVE_SITE_CONF_metastore_storage_schema_reader_impl=org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader +BEELINE_SITE_CONF_beeline_hs2_jdbc_url_tcpUrl=jdbc:hive2://${HOST}:${HS_PORT}/default;user=hdfs;password=hive +BEELINE_SITE_CONF_beeline_hs2_jdbc_url_httpUrl=jdbc:hive2://${HOST}:${HS_PORT}/default;user=hdfs;password=hive + + +CORE_CONF_fs_defaultFS=hdfs://${HOST}:${FS_PORT} +CORE_CONF_hadoop_http_staticuser_user=root +CORE_CONF_hadoop_proxyuser_hue_hosts=* +CORE_CONF_hadoop_proxyuser_hue_groups=* + +HDFS_CONF_dfs_webhdfs_enabled=true +HDFS_CONF_dfs_permissions_enabled=false +HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false +HDFS_CONF_dfs_datanode_address=${HOST}:${DFS_DN_PORT} +HDFS_CONF_dfs_datanode_http_address=${HOST}:${DFS_DN_HTTP_PORT} +HDFS_CONF_dfs_datanode_ipc_address=${HOST}:${DFS_DN_IPC_PORT} +HDFS_CONF_dfs_namenode_http___address=${HOST}:${DFS_NN_HTTP_PORT} +YARN_CONF_yarn_log___aggregation___enable=true +YARN_CONF_yarn_resourcemanager_recovery_enabled=true +YARN_CONF_yarn_resourcemanager_store_class=org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore +YARN_CONF_yarn_resourcemanager_fs_state___store_uri=/rmstate +YARN_CONF_yarn_nodemanager_remote___app___log___dir=/var/log/hadoop-yarn/apps +YARN_CONF_yarn_log_server_url=http://${HOST}:${YARM_LOG_SERVER_PORT}/jobhistory/logs +YARN_CONF_yarn_timeline___service_enabled=false +YARN_CONF_yarn_timeline___service_generic___application___history_enabled=true +YARN_CONF_yarn_resourcemanager_system___metrics___publisher_enabled=true +YARN_CONF_yarn_resourcemanager_hostname=${HOST} +MAPRED_CONF_mapreduce_shuffle_port=${MAPREDUCE_SHUFFLE_PORT} +YARN_CONF_yarn_timeline___service_hostname=${HOST} +YARN_CONF_yarn_resourcemanager_address=${HOST}:${YARN_RM_PORT} +YARN_CONF_yarn_resourcemanager_scheduler_address=${HOST}:${YARN_RM_SCHEDULER_PORT} +YARN_CONF_yarn_resourcemanager_resource___tracker_address=${HOST}:${YARN_RM_TRACKER_PORT} +YARN_CONF_yarn_resourcemanager_admin_address=${HOST}:${YARN_RM_ADMIN_PORT} +YARN_CONF_yarn_resourcemanager_webapp_address=${HOST}:${YARN_RM_WEBAPP_PORT} +YARN_CONF_yarn_nodemanager_localizer_address=${HOST}:${YARN_NM_LOCAL_PORT} +YARN_CONF_yarn_nodemanager_webapp_address=${HOST}:${YARN_NM_WEBAPP_PORT} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh b/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh index 473d7ceaeb6166..515f37e36ac9e3 100755 --- a/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh +++ b/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh @@ -32,4 +32,3 @@ if test -d "${HEALTH_D}"; then "${health_script}" &>> /var/log/container-health.log || exit 1 done fi -exit 0 diff --git a/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check-2.sh b/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check-2.sh index 854524dac1fcff..7545969bc47d65 100755 --- a/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check-2.sh +++ b/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check-2.sh @@ -17,4 +17,4 @@ # under the License. kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master-2@OTHERREALM.COM -beeline -u "jdbc:hive2://localhost:10000/default;principal=hive/hadoop-master-2@OTHERREALM.COM" -e "show databases;" \ No newline at end of file +beeline -u "jdbc:hive2://localhost:16000/default;principal=hive/hadoop-master-2@OTHERREALM.COM" -e "show databases;" \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check.sh b/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check.sh index 4d3d86f69a25c4..ab464b5233b31a 100755 --- a/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check.sh +++ b/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check.sh @@ -17,4 +17,4 @@ # under the License. kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master@LABS.TERADATA.COM -beeline -u "jdbc:hive2://localhost:10000/default;principal=hive/hadoop-master@LABS.TERADATA.COM" -e "show databases;" \ No newline at end of file +beeline -u "jdbc:hive2://localhost:15000/default;principal=hive/hadoop-master@LABS.TERADATA.COM" -e "show databases;" \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/kerberos/health-checks/hadoop-health-check.sh b/docker/thirdparties/docker-compose/kerberos/health-checks/supervisorctl-check.sh similarity index 100% rename from docker/thirdparties/docker-compose/kerberos/health-checks/hadoop-health-check.sh rename to docker/thirdparties/docker-compose/kerberos/health-checks/supervisorctl-check.sh diff --git a/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl b/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl index e635ed6bb27f34..9a1520b74db98e 100644 --- a/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl +++ b/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl @@ -16,68 +16,55 @@ # under the License. version: "3" services: - hive-krb: + hive-krb1: image: doristhirdpartydocker/trinodb:hdp3.1-hive-kerberized_96 - container_name: doris--kerberos1 + container_name: doris-${CONTAINER_UID}-kerberos1 volumes: + - ../common:/usr/local/common - ./two-kerberos-hives:/keytabs - ./sql:/usr/local/sql - ./common/hadoop/apply-config-overrides.sh:/etc/hadoop-init.d/00-apply-config-overrides.sh - ./common/hadoop/hadoop-run.sh:/usr/local/hadoop-run.sh - ./health-checks/health.sh:/usr/local/health.sh - - ./health-checks/hadoop-health-check.sh:/etc/health.d/hadoop-health-check.sh + - ./health-checks/supervisorctl-check.sh:/etc/health.d/supervisorctl-check.sh - ./health-checks/hive-health-check.sh:/etc/health.d/hive-health-check.sh - ./entrypoint-hive-master.sh:/usr/local/entrypoint-hive-master.sh + - ./conf/kerberos1/my.cnf:/etc/my.cnf + - ./conf/kerberos1/kdc.conf:/var/kerberos/krb5kdc/kdc.conf + - ./conf/kerberos1/krb5.conf:/etc/krb5.conf hostname: hadoop-master - entrypoint: /usr/local/entrypoint-hive-master.sh + entrypoint: /usr/local/entrypoint-hive-master.sh 1 healthcheck: - test: ["CMD", "ls", "/mnt/SUCCESS"] - interval: 20s - timeout: 60s + test: ["CMD", "ls", "/tmp/SUCCESS"] + interval: 5s + timeout: 10s retries: 120 - ports: - - "5806:5006" - - "8820:8020" - - "8842:8042" - - "9800:9000" - - "9883:9083" - - "18000:10000" - networks: - doris--krb_net: - ipv4_address: 172.31.71.25 - + network_mode: "host" + env_file: + - ./hadoop-hive-1.env hive-krb2: image: doristhirdpartydocker/trinodb:hdp3.1-hive-kerberized-2_96 - container_name: doris--kerberos2 + container_name: doris-${CONTAINER_UID}-kerberos2 hostname: hadoop-master-2 volumes: + - ../common:/usr/local/common - ./two-kerberos-hives:/keytabs - ./sql:/usr/local/sql - ./common/hadoop/apply-config-overrides.sh:/etc/hadoop-init.d/00-apply-config-overrides.sh - ./common/hadoop/hadoop-run.sh:/usr/local/hadoop-run.sh - ./health-checks/health.sh:/usr/local/health.sh - - ./health-checks/hadoop-health-check.sh:/etc/health.d/hadoop-health-check.sh + - ./health-checks/supervisorctl-check.sh:/etc/health.d/supervisorctl-check.sh - ./health-checks/hive-health-check-2.sh:/etc/health.d/hive-health-check-2.sh - - ./entrypoint-hive-master-2.sh:/usr/local/entrypoint-hive-master-2.sh - entrypoint: /usr/local/entrypoint-hive-master-2.sh + - ./entrypoint-hive-master.sh:/usr/local/entrypoint-hive-master.sh + - ./conf/kerberos2/my.cnf:/etc/my.cnf + - ./conf/kerberos2/kdc.conf:/var/kerberos/krb5kdc/kdc.conf + - ./conf/kerberos2/krb5.conf:/etc/krb5.conf + entrypoint: /usr/local/entrypoint-hive-master.sh 2 healthcheck: - test: ["CMD", "ls", "/mnt/SUCCESS"] - interval: 20s - timeout: 60s + test: ["CMD", "ls", "/tmp/SUCCESS"] + interval: 5s + timeout: 10s retries: 120 - ports: - - "15806:5006" - - "18820:8020" - - "18842:8042" - - "19800:9000" - - "19883:9083" - - "18800:10000" - networks: - doris--krb_net: - ipv4_address: 172.31.71.26 - -networks: - doris--krb_net: - ipam: - config: - - subnet: 172.31.71.0/24 + network_mode: "host" + env_file: + - ./hadoop-hive-2.env \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/kerberos/kerberos1_settings.env b/docker/thirdparties/docker-compose/kerberos/kerberos1_settings.env new file mode 100644 index 00000000000000..aaf4c639fa8509 --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/kerberos1_settings.env @@ -0,0 +1,47 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Change this to a specific string. +# Do not use "_" or other sepcial characters, only number and alphabeta. +# NOTICE: change this uid will modify hive-*.yaml + +export HOST="hadoop-master" +export FS_PORT=8520 +export HMS_PORT=9583 +export HS_PORT=15000 +export MYSQL_PORT=3356 +export DFS_DN_PORT=9566 +export DFS_DN_HTTP_PORT=9564 +export DFS_DN_IPC_PORT=9567 +export DFS_NN_HTTP_PORT=9570 +export YARM_LOG_SERVER_PORT=8588 +export YARN_RM_PORT=8532 +export YARN_RM_SCHEDULER_PORT=8530 +export YARN_RM_TRACKER_PORT=8531 +export YARN_RM_ADMIN_PORT=8533 +export YARN_RM_WEBAPP_PORT=8589 +export YARN_NM_LOCAL_PORT=8540 +export YARN_NM_WEBAPP_PORT=8542 +export MAPREDUCE_SHUFFLE_PORT=13562 +export KADMIND_PORT=5464 +export KDC_PORT1=5588 +export KDC_PORT2=5589 +export KADMIND_PORT1=5749 +export KADMIND_PORT2=5750 +export KPASSWD_PORT1=5464 +export KPASSWD_PORT2=5465 \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/kerberos/kerberos2_settings.env b/docker/thirdparties/docker-compose/kerberos/kerberos2_settings.env new file mode 100644 index 00000000000000..fca68d60162091 --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/kerberos2_settings.env @@ -0,0 +1,43 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Change this to a specific string. +# Do not use "_" or other sepcial characters, only number and alphabeta. +# NOTICE: change this uid will modify hive-*.yaml + +export HOST="hadoop-master-2" +export FS_PORT=8620 +export HMS_PORT=9683 +export HS_PORT=16000 +export MYSQL_PORT=3366 +export DFS_DN_PORT=9666 +export DFS_DN_HTTP_PORT=9664 +export DFS_DN_IPC_PORT=9667 +export DFS_NN_HTTP_PORT=9670 +export YARM_LOG_SERVER_PORT=8688 +export YARN_RM_PORT=8632 +export YARN_RM_SCHEDULER_PORT=8630 +export YARN_RM_TRACKER_PORT=8631 +export YARN_RM_ADMIN_PORT=8633 +export YARN_RM_WEBAPP_PORT=8689 +export YARN_NM_LOCAL_PORT=8640 +export YARN_NM_WEBAPP_PORT=8642 +export MAPREDUCE_SHUFFLE_PORT=13662 +export KDC_PORT1=6688 +export KADMIND_PORT1=6749 +export KPASSWD_PORT1=6464 \ No newline at end of file diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh index e3769025fec62d..a581baa6cfa1e7 100755 --- a/docker/thirdparties/run-thirdparties-docker.sh +++ b/docker/thirdparties/run-thirdparties-docker.sh @@ -582,8 +582,22 @@ start_lakesoul() { start_kerberos() { echo "RUN_KERBEROS" - cp "${ROOT}"/docker-compose/kerberos/kerberos.yaml.tpl "${ROOT}"/docker-compose/kerberos/kerberos.yaml - sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/kerberos/kerberos.yaml + eth_name=$(ifconfig -a | grep -E "^eth[0-9]" | sort -k1.4n | awk -F ':' '{print $1}' | head -n 1) + IP_HOST=$(ifconfig "${eth_name}" | grep inet | grep -v 127.0.0.1 | grep -v inet6 | awk '{print $2}' | tr -d "addr:" | head -n 1) + export IP_HOST=${IP_HOST} + export CONTAINER_UID=${CONTAINER_UID} + envsubst <"${ROOT}"/docker-compose/kerberos/kerberos.yaml.tpl >"${ROOT}"/docker-compose/kerberos/kerberos.yaml + for i in {1..2}; do + . "${ROOT}"/docker-compose/kerberos/kerberos${i}_settings.env + envsubst <"${ROOT}"/docker-compose/kerberos/hadoop-hive.env.tpl >"${ROOT}"/docker-compose/kerberos/hadoop-hive-${i}.env + envsubst <"${ROOT}"/docker-compose/kerberos/conf/my.cnf.tpl > "${ROOT}"/docker-compose/kerberos/conf/kerberos${i}/my.cnf + envsubst <"${ROOT}"/docker-compose/kerberos/conf/kerberos${i}/kdc.conf.tpl > "${ROOT}"/docker-compose/kerberos/conf/kerberos${i}/kdc.conf + envsubst <"${ROOT}"/docker-compose/kerberos/conf/kerberos${i}/krb5.conf.tpl > "${ROOT}"/docker-compose/kerberos/conf/kerberos${i}/krb5.conf + done + sudo chmod a+w /etc/hosts + sudo sed -i "1i${IP_HOST} hadoop-master" /etc/hosts + sudo sed -i "1i${IP_HOST} hadoop-master-2" /etc/hosts + sudo cp "${ROOT}"/docker-compose/kerberos/kerberos.yaml.tpl "${ROOT}"/docker-compose/kerberos/kerberos.yaml sudo docker compose -f "${ROOT}"/docker-compose/kerberos/kerberos.yaml down sudo rm -rf "${ROOT}"/docker-compose/kerberos/data if [[ "${STOP}" -ne 1 ]]; then @@ -591,15 +605,11 @@ start_kerberos() { rm -rf "${ROOT}"/docker-compose/kerberos/two-kerberos-hives/*.keytab rm -rf "${ROOT}"/docker-compose/kerberos/two-kerberos-hives/*.jks rm -rf "${ROOT}"/docker-compose/kerberos/two-kerberos-hives/*.conf - sudo docker compose -f "${ROOT}"/docker-compose/kerberos/kerberos.yaml up -d + sudo docker compose -f "${ROOT}"/docker-compose/kerberos/kerberos.yaml up -d --wait sudo rm -f /keytabs sudo ln -s "${ROOT}"/docker-compose/kerberos/two-kerberos-hives /keytabs sudo cp "${ROOT}"/docker-compose/kerberos/common/conf/doris-krb5.conf /keytabs/krb5.conf sudo cp "${ROOT}"/docker-compose/kerberos/common/conf/doris-krb5.conf /etc/krb5.conf - - sudo chmod a+w /etc/hosts - echo '172.31.71.25 hadoop-master' >> /etc/hosts - echo '172.31.71.26 hadoop-master-2' >> /etc/hosts sleep 2 fi } diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 4a6833455560b3..8bda7f6e754c2f 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -3281,6 +3281,14 @@ public static int metaServiceRpcRetryTimes() { @ConfField(mutable = true, description = {"存算分离模式下commit阶段等锁超时时间，默认5s"}) public static int try_commit_lock_timeout_seconds = 5; + @ConfField(mutable = true, description = {"是否在事务提交时对所有表启用提交锁。设置为 true 时，所有表都会使用提交锁。" + + "设置为 false 时，仅对 Merge-On-Write 表使用提交锁。默认值为 true。", + "Whether to enable commit lock for all tables during transaction commit." + + "If true, commit lock will be applied to all tables." + + "If false, commit lock will only be applied to Merge-On-Write tables." + + "Default value is true." }) + public static boolean enable_commit_lock_for_all_tables = true; + @ConfField(mutable = true, description = {"存算分离模式下是否开启大事务提交，默认false"}) public static boolean enable_cloud_txn_lazy_commit = false; diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 8c21461f1e68b7..d24a1f74f2c7ce 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -224,6 +224,8 @@ supportedAlterStatement dropRollupClause (COMMA dropRollupClause)* #alterTableDropRollup | ALTER TABLE name=multipartIdentifier SET LEFT_PAREN propertyItemList RIGHT_PAREN #alterTableProperties + | ALTER DATABASE name=identifier SET (DATA | REPLICA | TRANSACTION) + QUOTA (quota=identifier | INTEGER_VALUE) #alterDatabaseSetQuota | ALTER SYSTEM RENAME COMPUTE GROUP name=identifier newName=identifier #alterSystemRenameComputeGroup ; @@ -593,8 +595,6 @@ privilegeList unsupportedAlterStatement : ALTER SYSTEM alterSystemClause #alterSystem - | ALTER DATABASE name=identifier SET (DATA |REPLICA | TRANSACTION) - QUOTA INTEGER_VALUE identifier? #alterDatabaseSetQuota | ALTER DATABASE name=identifier SET PROPERTIES LEFT_PAREN propertyItemList RIGHT_PAREN #alterDatabaseProperties | ALTER CATALOG name=identifier SET PROPERTIES diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 7046a71c63bdee..1c9cf2452ce321 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -29,7 +29,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import org.apache.doris.analysis.AlterDatabaseQuotaStmt.QuotaType; +import org.apache.doris.alter.QuotaType; import org.apache.doris.analysis.ColumnNullableType; import org.apache.doris.analysis.SetOperationStmt.Qualifier; import org.apache.doris.analysis.SetOperationStmt.Operation; diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java index c8cd28c8617506..3b59c3bac9b056 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java @@ -699,6 +699,10 @@ private void processReplaceTable(Database db, OlapTable origTable, List tableList, long executeCommitTxnRequest(commitTxnRequest, transactionId, false, null); } - // add some log and get commit lock, mainly used for mow tables + private List getTablesNeedCommitLock(List
tableList) { + if (Config.enable_commit_lock_for_all_tables) { + // If enabled, lock all tables + return tableList.stream() + .sorted(Comparator.comparingLong(Table::getId)) + .collect(Collectors.toList()); + } else { + // If disabled, only lock MOW tables + return tableList.stream() + .filter(table -> table instanceof OlapTable && ((OlapTable) table).getEnableUniqueKeyMergeOnWrite()) + .sorted(Comparator.comparingLong(Table::getId)) + .collect(Collectors.toList()); + } + } + private void beforeCommitTransaction(List
tableList, long transactionId, long timeoutMillis) throws UserException { for (int i = 0; i < tableList.size(); i++) { @@ -1180,29 +1194,21 @@ private void beforeCommitTransaction(List
tableList, long transactionId, } } - // Get tables that require commit lock - only MOW tables need this: - // 1. Filter to keep only OlapTables with MOW enabled - // 2. Sort by table ID to maintain consistent locking order and prevent deadlocks - List
mowTableList = tableList.stream() - .filter(table -> table instanceof OlapTable && ((OlapTable) table).getEnableUniqueKeyMergeOnWrite()) - .sorted(Comparator.comparingLong(Table::getId)) - .collect(Collectors.toList()); - increaseWaitingLockCount(mowTableList); - if (!MetaLockUtils.tryCommitLockTables(mowTableList, timeoutMillis, TimeUnit.MILLISECONDS)) { - decreaseWaitingLockCount(mowTableList); + List
tablesToLock = getTablesNeedCommitLock(tableList); + increaseWaitingLockCount(tablesToLock); + if (!MetaLockUtils.tryCommitLockTables(tablesToLock, timeoutMillis, TimeUnit.MILLISECONDS)) { + decreaseWaitingLockCount(tablesToLock); // DELETE_BITMAP_LOCK_ERR will be retried on be throw new UserException(InternalErrorCode.DELETE_BITMAP_LOCK_ERR, "get table cloud commit lock timeout, tableList=(" - + StringUtils.join(mowTableList, ",") + ")"); + + StringUtils.join(tablesToLock, ",") + ")"); } } private void afterCommitTransaction(List
tableList) { - List
mowTableList = tableList.stream() - .filter(table -> table instanceof OlapTable && ((OlapTable) table).getEnableUniqueKeyMergeOnWrite()) - .collect(Collectors.toList()); - decreaseWaitingLockCount(mowTableList); - MetaLockUtils.commitUnlockTables(mowTableList); + List
tablesToUnlock = getTablesNeedCommitLock(tableList); + decreaseWaitingLockCount(tablesToUnlock); + MetaLockUtils.commitUnlockTables(tablesToUnlock); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 5b1ee38c64da08..1508675b2e28b3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -17,13 +17,12 @@ package org.apache.doris.datasource; +import org.apache.doris.alter.QuotaType; import org.apache.doris.analysis.AddPartitionClause; import org.apache.doris.analysis.AddPartitionLikeClause; import org.apache.doris.analysis.AddRollupClause; import org.apache.doris.analysis.AlterClause; import org.apache.doris.analysis.AlterDatabasePropertyStmt; -import org.apache.doris.analysis.AlterDatabaseQuotaStmt; -import org.apache.doris.analysis.AlterDatabaseQuotaStmt.QuotaType; import org.apache.doris.analysis.AlterMultiPartitionClause; import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.ColumnDef; @@ -762,21 +761,19 @@ public void replayRecoverDatabase(RecoverInfo info) { LOG.info("replay recover db[{}]", dbId); } - public void alterDatabaseQuota(AlterDatabaseQuotaStmt stmt) throws DdlException { - String dbName = stmt.getDbName(); - Database db = (Database) getDbOrDdlException(dbName); - QuotaType quotaType = stmt.getQuotaType(); + public void alterDatabaseQuota(String dbName, QuotaType quotaType, long quotaValue) throws DdlException { + Database db = getDbOrDdlException(dbName); db.writeLockOrDdlException(); try { if (quotaType == QuotaType.DATA) { - db.setDataQuota(stmt.getQuota()); + db.setDataQuota(quotaValue); } else if (quotaType == QuotaType.REPLICA) { - db.setReplicaQuota(stmt.getQuota()); + db.setReplicaQuota(quotaValue); } else if (quotaType == QuotaType.TRANSACTION) { - db.setTransactionQuotaSize(stmt.getQuota()); + db.setTransactionQuotaSize(quotaValue); } - long quota = stmt.getQuota(); - DatabaseInfo dbInfo = new DatabaseInfo(dbName, "", quota, quotaType); + + DatabaseInfo dbInfo = new DatabaseInfo(dbName, "", quotaValue, quotaType); Env.getCurrentEnv().getEditLog().logAlterDb(dbInfo); } finally { db.writeUnlock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java index f3939cb47eafd8..125b8ce38deecd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java @@ -261,6 +261,10 @@ public void dropTable(Table table) { public void alterTable(Table table, String oldTableName) { BaseTableInfo baseTableInfo = new BaseTableInfo(table); baseTableInfo.setTableName(oldTableName); + if (table instanceof MTMV) { + removeMTMV(baseTableInfo); + refreshMTMVCache(((MTMV) table).getRelation(), new BaseTableInfo(table)); + } processBaseTableChange(baseTableInfo, "The base table has been updated:"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 476ad72285c1e9..2232f01bb3020b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.parser; +import org.apache.doris.alter.QuotaType; import org.apache.doris.analysis.ArithmeticExpr.Operator; import org.apache.doris.analysis.BrokerDesc; import org.apache.doris.analysis.ColumnNullableType; @@ -72,6 +73,7 @@ import org.apache.doris.nereids.DorisParser.AlterCatalogCommentContext; import org.apache.doris.nereids.DorisParser.AlterCatalogRenameContext; import org.apache.doris.nereids.DorisParser.AlterDatabaseRenameContext; +import org.apache.doris.nereids.DorisParser.AlterDatabaseSetQuotaContext; import org.apache.doris.nereids.DorisParser.AlterMTMVContext; import org.apache.doris.nereids.DorisParser.AlterMultiPartitionClauseContext; import org.apache.doris.nereids.DorisParser.AlterRoleContext; @@ -624,6 +626,7 @@ import org.apache.doris.nereids.trees.plans.commands.UnsupportedCommand; import org.apache.doris.nereids.trees.plans.commands.UpdateCommand; import org.apache.doris.nereids.trees.plans.commands.alter.AlterDatabaseRenameCommand; +import org.apache.doris.nereids.trees.plans.commands.alter.AlterDatabaseSetQuotaCommand; import org.apache.doris.nereids.trees.plans.commands.clean.CleanLabelCommand; import org.apache.doris.nereids.trees.plans.commands.info.AddColumnOp; import org.apache.doris.nereids.trees.plans.commands.info.AddColumnsOp; @@ -5464,6 +5467,30 @@ public LogicalPlan visitShowConvertLsc(ShowConvertLscContext ctx) { return new ShowConvertLSCCommand(databaseName); } + @Override + public Object visitAlterDatabaseSetQuota(AlterDatabaseSetQuotaContext ctx) { + String databaseName = Optional.ofNullable(ctx.name) + .map(ParseTree::getText).filter(s -> !s.isEmpty()) + .orElseThrow(() -> new ParseException("database name can not be null")); + String quota = Optional.ofNullable(ctx.quota) + .map(ParseTree::getText) + .orElseGet(() -> Optional.ofNullable(ctx.INTEGER_VALUE()) + .map(TerminalNode::getText) + .orElse(null)); + // Determine the quota type + QuotaType quotaType; + if (ctx.DATA() != null) { + quotaType = QuotaType.DATA; + } else if (ctx.REPLICA() != null) { + quotaType = QuotaType.REPLICA; + } else if (ctx.TRANSACTION() != null) { + quotaType = QuotaType.TRANSACTION; + } else { + quotaType = QuotaType.NONE; + } + return new AlterDatabaseSetQuotaCommand(databaseName, quotaType, quota); + } + @Override public LogicalPlan visitDropDatabase(DropDatabaseContext ctx) { boolean ifExists = ctx.EXISTS() != null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifySelfComparison.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifySelfComparison.java index dbd89964710de5..76e93c442034d4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifySelfComparison.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifySelfComparison.java @@ -54,7 +54,12 @@ public List> buildRules() { private Expression rewrite(ComparisonPredicate comparison) { Expression left = comparison.left(); - if (left.equals(comparison.right())) { + // expression maybe non-deterministic, but if it's foldable, then it still can simplify self comparison. + // for example, function `user()`, `current_timestamp()` are foldable and non-deterministic, + // then `user() = user()` and `current_timestamp() = current_timestamp()` can simplify to `TRUE`. + // function `random` is not foldable and non-deterministic, + // then `random(1, 10) = random(1, 10)` cann't simplify to `TRUE` + if (!left.containsNonfoldable() && left.equals(comparison.right())) { if (comparison instanceof EqualTo || comparison instanceof GreaterThanEqual || comparison instanceof LessThanEqual) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java index 64019818f5d1e7..5daf3bb254fc24 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java @@ -501,8 +501,8 @@ private Statistics computeOlapScan(OlapScan olapScan) { // mv is selected, return its estimated stats Optional optStats = cascadesContext.getStatementContext() .getStatistics(((Relation) olapScan).getRelationId()); - LOG.info("computeOlapScan optStats isPresent {}, tableRowCount is {}", - optStats.isPresent(), tableRowCount); + LOG.info("computeOlapScan optStats isPresent {}, tableRowCount is {}, table name is {}", + optStats.isPresent(), tableRowCount, olapTable.getQualifiedName()); if (optStats.isPresent()) { double selectedPartitionsRowCount = getSelectedPartitionRowCount(olapScan, tableRowCount); LOG.info("computeOlapScan optStats is {}, selectedPartitionsRowCount is {}", optStats.get(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExpressionTrait.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExpressionTrait.java index daaab359e8433a..6e33f2a5e113fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExpressionTrait.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExpressionTrait.java @@ -77,6 +77,13 @@ default boolean foldable() { return true; } + /** + * Identify the expression is containing non-foldable expr or not + */ + default boolean containsNonfoldable() { + return anyMatch(expr -> !((ExpressionTrait) expr).foldable()); + } + /** * Identify the expression itself is deterministic or not, default true */ @@ -85,7 +92,7 @@ default boolean isDeterministic() { } /** - * Identify the expression is containing deterministic expr or not + * Identify the expression is containing non-deterministic expr or not */ default boolean containsNondeterministic() { return anyMatch(expr -> !((ExpressionTrait) expr).isDeterministic()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java index cd1789b687efd0..2e54d7895dd921 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java @@ -187,6 +187,7 @@ public enum PlanType { ALTER_WORKLOAD_GROUP_COMMAND, ALTER_WORKLOAD_POLICY_COMMAND, ALTER_DATABASE_RENAME_COMMAND, + ALTER_DATABASE_SET_DATA_QUOTA_COMMAND, DROP_CATALOG_RECYCLE_BIN_COMMAND, DROP_ENCRYPTKEY_COMMAND, DROP_FILE_COMMAND, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/alter/AlterDatabaseSetQuotaCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/alter/AlterDatabaseSetQuotaCommand.java new file mode 100644 index 00000000000000..db1379d72a5789 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/alter/AlterDatabaseSetQuotaCommand.java @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.alter; + +import org.apache.doris.alter.QuotaType; +import org.apache.doris.catalog.Env; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.util.InternalDatabaseUtil; +import org.apache.doris.common.util.ParseUtil; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.commands.AlterCommand; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.StmtExecutor; + +import com.google.common.base.Strings; +import com.google.gson.annotations.SerializedName; + +/** + * Represents the command for ALTER DATABASE db_name SET (DATA |REPLICA | TRANSACTION) QUOTA quota_value. + */ +public class AlterDatabaseSetQuotaCommand extends AlterCommand { + + @SerializedName("db") + private String dbName; + + @SerializedName("qt") + private QuotaType quotaType; + + @SerializedName("qv") + private String quotaValue; + + @SerializedName("q") + private long quota; + + public AlterDatabaseSetQuotaCommand(String dbName, QuotaType quotaType, String quotaValue) { + super(PlanType.ALTER_DATABASE_SET_DATA_QUOTA_COMMAND); + this.dbName = dbName; + this.quotaType = quotaType; + this.quotaValue = quotaValue; + } + + @Override + public void doRun(ConnectContext ctx, StmtExecutor executor) throws Exception { + InternalDatabaseUtil.checkDatabase(dbName, ConnectContext.get()); + if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_DBACCESS_DENIED_ERROR, + ctx.getQualifiedUser(), dbName); + } + + if (Strings.isNullOrEmpty(dbName)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR); + } + if (quotaType == QuotaType.DATA) { + quota = ParseUtil.analyzeDataVolume(quotaValue); + } else if (quotaType == QuotaType.REPLICA) { + quota = ParseUtil.analyzeReplicaNumber(quotaValue); + } else if (quotaType == QuotaType.TRANSACTION) { + quota = ParseUtil.analyzeTransactionNumber(quotaValue); + } + Env.getCurrentInternalCatalog().alterDatabaseQuota(dbName, quotaType, quota); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitAlterDatabaseSetQuotaCommand(this, context); + } + + public String toSql() { + return "ALTER DATABASE " + dbName + " SET " + + quotaType.name() + + " QUOTA " + quotaValue; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java index c65ce4a282ddf8..76c72f82f90552 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java @@ -28,6 +28,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.profile.ProfileManager.ProfileType; import org.apache.doris.common.util.DebugUtil; +import org.apache.doris.common.util.MetaLockUtils; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.jdbc.JdbcExternalTable; @@ -72,6 +73,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.Comparator; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -184,7 +186,9 @@ public AbstractInsertExecutor initPlan(ConnectContext ctx, StmtExecutor stmtExec // lock after plan and check does table's schema changed to ensure we lock table order by id. TableIf newestTargetTableIf = RelationUtil.getTable(qualifiedTargetTableName, ctx.getEnv()); - newestTargetTableIf.readLock(); + List targetTables = Lists.newArrayList(targetTableIf, newestTargetTableIf); + targetTables.sort(Comparator.comparing(TableIf::getId)); + MetaLockUtils.readLockTables(targetTables); try { if (targetTableIf.getId() != newestTargetTableIf.getId()) { LOG.warn("insert plan failed {} times. query id is {}. table id changed from {} to {}", @@ -205,9 +209,9 @@ public AbstractInsertExecutor initPlan(ConnectContext ctx, StmtExecutor stmtExec buildResult.physicalSink ); } - newestTargetTableIf.readUnlock(); + MetaLockUtils.readUnlockTables(targetTables); } catch (Throwable e) { - newestTargetTableIf.readUnlock(); + MetaLockUtils.readUnlockTables(targetTables); // the abortTxn in onFail need to acquire table write lock if (insertExecutor != null) { insertExecutor.onFail(e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java index c849f9ce1f8fe2..5c33ad1d7a4101 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java @@ -154,6 +154,7 @@ import org.apache.doris.nereids.trees.plans.commands.UnsupportedCommand; import org.apache.doris.nereids.trees.plans.commands.UpdateCommand; import org.apache.doris.nereids.trees.plans.commands.alter.AlterDatabaseRenameCommand; +import org.apache.doris.nereids.trees.plans.commands.alter.AlterDatabaseSetQuotaCommand; import org.apache.doris.nereids.trees.plans.commands.clean.CleanLabelCommand; import org.apache.doris.nereids.trees.plans.commands.insert.BatchInsertIntoTableCommand; import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand; @@ -759,6 +760,10 @@ default R visitAlterDatabaseRenameCommand(AlterDatabaseRenameCommand alterDataba return visitCommand(alterDatabaseRenameCommand, context); } + default R visitAlterDatabaseSetQuotaCommand(AlterDatabaseSetQuotaCommand alterDatabaseSetQuotaCommand, C context) { + return visitCommand(alterDatabaseSetQuotaCommand, context); + } + default R visitDropDatabaseCommand(DropDatabaseCommand dropDatabaseCommand, C context) { return visitCommand(dropDatabaseCommand, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/DatabaseInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/DatabaseInfo.java index 966444c11c4fba..a48ce474b370e4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/DatabaseInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/DatabaseInfo.java @@ -17,7 +17,7 @@ package org.apache.doris.persist; -import org.apache.doris.analysis.AlterDatabaseQuotaStmt.QuotaType; +import org.apache.doris.alter.QuotaType; import org.apache.doris.catalog.BinlogConfig; import org.apache.doris.catalog.Database.DbState; import org.apache.doris.catalog.Env; diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/AlterDatabaseQuotaStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/AlterDatabaseQuotaStmtTest.java index eb04e7615a9100..745cc95bcec2cc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/AlterDatabaseQuotaStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/AlterDatabaseQuotaStmtTest.java @@ -17,7 +17,7 @@ package org.apache.doris.analysis; -import org.apache.doris.analysis.AlterDatabaseQuotaStmt.QuotaType; +import org.apache.doris.alter.QuotaType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; import org.apache.doris.mysql.privilege.AccessControllerManager; diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java new file mode 100644 index 00000000000000..17ec145f583221 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.MTMV; +import org.apache.doris.catalog.Table; +import org.apache.doris.common.DdlException; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Set; + + +public class AlterMTMVTest extends TestWithFeService { + + @Test + public void testAlterMTMV() throws Exception { + createDatabaseAndUse("test"); + + createTable("CREATE TABLE `stu` (`sid` int(32) NULL, `sname` varchar(32) NULL)\n" + + "ENGINE=OLAP\n" + + "DUPLICATE KEY(`sid`)\n" + + "DISTRIBUTED BY HASH(`sid`) BUCKETS 1\n" + + "PROPERTIES ('replication_allocation' = 'tag.location.default: 1')"); + + createMvByNereids("CREATE MATERIALIZED VIEW mv_a BUILD IMMEDIATE REFRESH COMPLETE ON COMMIT\n" + + "DISTRIBUTED BY HASH(`sid`) BUCKETS 1\n" + + "PROPERTIES ('replication_allocation' = 'tag.location.default: 1') " + + "AS select * from stu limit 1"); + + alterMv("ALTER MATERIALIZED VIEW mv_a RENAME mv_b"); + + MTMVRelationManager relationManager = Env.getCurrentEnv().getMtmvService().getRelationManager(); + Table table = Env.getCurrentInternalCatalog().getDb("test").get().getTableOrMetaException("stu"); + Set allMTMVs = relationManager.getAllMTMVs(Lists.newArrayList(new BaseTableInfo(table))); + boolean hasMvA = false; + boolean hasMvB = false; + for (MTMV mtmv : allMTMVs) { + if ("mv_a".equals(mtmv.getName())) { + hasMvA = true; + } + if ("mv_b".equals(mtmv.getName())) { + hasMvB = true; + } + } + Assertions.assertFalse(hasMvA); + Assertions.assertTrue(hasMvB); + + + createTable("CREATE TABLE `stu1` (`sid` int(32) NULL, `sname` varchar(32) NULL)\n" + + "ENGINE=OLAP\n" + + "DUPLICATE KEY(`sid`)\n" + + "DISTRIBUTED BY HASH(`sid`) BUCKETS 1\n" + + "PROPERTIES ('replication_allocation' = 'tag.location.default: 1')"); + + DdlException exception = Assertions.assertThrows(DdlException.class, () -> + alterTableSync("ALTER TABLE stu1 REPLACE WITH TABLE mv_b PROPERTIES('swap' = 'true')")); + Assertions.assertEquals("errCode = 2, detailMessage = replace table[mv_b] cannot be a materialized view", exception.getMessage()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifySelfComparisonTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifySelfComparisonTest.java index be59404b46786b..28a5ed3b7c3329 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifySelfComparisonTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifySelfComparisonTest.java @@ -32,12 +32,20 @@ void testRewrite() { ExpressionRewrite.bottomUp(SimplifySelfComparison.INSTANCE) )); + // foldable, cast assertRewriteAfterTypeCoercion("TA + TB = TA + TB", "NOT ((TA + TB) IS NULL) OR NULL"); assertRewriteAfterTypeCoercion("TA + TB >= TA + TB", "NOT ((TA + TB) IS NULL) OR NULL"); assertRewriteAfterTypeCoercion("TA + TB <= TA + TB", "NOT ((TA + TB) IS NULL) OR NULL"); assertRewriteAfterTypeCoercion("TA + TB <=> TA + TB", "TRUE"); assertRewriteAfterTypeCoercion("TA + TB > TA + TB", "(TA + TB) IS NULL AND NULL"); assertRewriteAfterTypeCoercion("TA + TB < TA + TB", "(TA + TB) IS NULL AND NULL"); + assertRewriteAfterTypeCoercion("DAYS_ADD(CA, 7) <=> DAYS_ADD(CA, 7)", "TRUE"); + assertRewriteAfterTypeCoercion("USER() = USER()", "TRUE"); + assertRewriteAfterTypeCoercion("CURRENT_TIMESTAMP() = CURRENT_TIMESTAMP()", "TRUE"); + + // not foldable, not cast + assertRewriteAfterTypeCoercion("random(5, 10) = random(5, 10)", "random(5, 10) = random(5, 10)"); + assertRewriteAfterTypeCoercion("random(5, 10) + 100 = random(5, 10) + 100", "random(5, 10) + 100 = random(5, 10) + 100"); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/DatabaseInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/DatabaseInfoTest.java index 3edd0b21ffb4c3..c965cbd995055e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/DatabaseInfoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/DatabaseInfoTest.java @@ -17,7 +17,7 @@ package org.apache.doris.persist; -import org.apache.doris.analysis.AlterDatabaseQuotaStmt.QuotaType; +import org.apache.doris.alter.QuotaType; import org.apache.doris.common.AnalysisException; import org.junit.Assert; diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java index c24703bf6f6fa3..35ac71314d414e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java +++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java @@ -66,6 +66,7 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.plans.commands.AddConstraintCommand; +import org.apache.doris.nereids.trees.plans.commands.AlterMTMVCommand; import org.apache.doris.nereids.trees.plans.commands.CreateMTMVCommand; import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand; import org.apache.doris.nereids.trees.plans.commands.DropConstraintCommand; @@ -621,6 +622,11 @@ public void createDatabase(String db) throws Exception { Env.getCurrentEnv().createDb(createDbStmt); } + public void createDatabaseAndUse(String db) throws Exception { + createDatabase(db); + useDatabase(db); + } + public void createDatabaseWithSql(String createDbSql) throws Exception { CreateDbStmt createDbStmt = (CreateDbStmt) parseAndAnalyzeStmt(createDbSql); Env.getCurrentEnv().createDb(createDbStmt); @@ -845,6 +851,19 @@ protected void createMv(String sql) throws Exception { Thread.sleep(100); } + protected void alterMv(String sql) throws Exception { + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan parsed = nereidsParser.parseSingle(sql); + StmtExecutor stmtExecutor = new StmtExecutor(connectContext, sql); + if (parsed instanceof AlterMTMVCommand) { + ((AlterMTMVCommand) parsed).run(connectContext, stmtExecutor); + } + checkAlterJob(); + // waiting table state to normal + Thread.sleep(1000); + } + + protected void createMvByNereids(String sql) throws Exception { new MockUp() { @Mock diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 70db19bfa33581..9ab20842c25e8f 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -392,18 +392,21 @@ struct TRuntimeFilterTargetParamsV2 { } struct TRuntimeFilterParams { - // Runtime filter merge instance address + // Runtime filter merge instance address. Used if this filter has a remote target 1: optional Types.TNetworkAddress runtime_filter_merge_addr // deprecated 2: optional map> rid_to_target_param // Runtime filter ID to the runtime filter desc + // Used if this filter has a remote target 3: optional map rid_to_runtime_filter // Number of Runtime filter producers + // Used if this filter has a remote target 4: optional map runtime_filter_builder_num + // Used if this filter has a remote target 5: optional map> rid_to_target_paramv2 } diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index be920c2baf50f6..c7f8d247d7c1b1 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -1252,7 +1252,7 @@ struct TRuntimeFilterDesc { // The order of Expr in join predicate 3: required i32 expr_order - // Map of target node id to the target expr + // Map of target node id to the target expr. Used by consumer 4: required map planId_to_target_expr // Indicates if the source join node of this filter is a broadcast or diff --git a/regression-test/data/external_table_p0/tvf/compress/test_tvf.csv.snappy b/regression-test/data/external_table_p0/tvf/compress/test_tvf.csv.snappy index 9ac2b7ae29977e..054613c514691b 100644 Binary files a/regression-test/data/external_table_p0/tvf/compress/test_tvf.csv.snappy and b/regression-test/data/external_table_p0/tvf/compress/test_tvf.csv.snappy differ diff --git a/regression-test/data/external_table_p0/tvf/test_local_tvf_compression.out b/regression-test/data/external_table_p0/tvf/test_local_tvf_compression.out index 8120427ea6cec5..5f1a4f5d463b93 100644 --- a/regression-test/data/external_table_p0/tvf/test_local_tvf_compression.out +++ b/regression-test/data/external_table_p0/tvf/test_local_tvf_compression.out @@ -123,28 +123,28 @@ 2023-09-18 7 -- !snappy_1 -- -1 694832 buHDwfGeNHfpRFdNaogneddi 2024-02-09 4.899588807225554 -10 218729 goZsLvvWFOIjlzSAitC 2024-06-10 4.137732740231178 -100 813423 zICskqgcdPc 2024-03-23 8.486529018746493 -1000 612650 RzOXeYpKOmuJOogUyeIEDNDmvq 2023-12-05 7.8741752707933435 -1001 29486 WoUAFJFuJNnwyqMnoDhX 2024-03-11 9.758244908785949 -1002 445363 OdTEeeWtxfcRwx 2024-08-01 0.3934945460194128 -1003 707035 JAYnKxusVpGzYueACf 2023-11-14 5.377110182643222 -1004 227858 JIFyjKzmbjkt 2024-03-24 5.748037621519263 -1005 539305 PlruLkSUSXZgaHafFriklrhCi 2023-11-08 4.122635188836725 -1006 145518 KCwqEcSCGuXrHerwn 2024-06-22 8.482290064407216 -1007 939028 KzXhEMelsKVLbDMsEKh 2024-01-01 8.144449761594585 -1008 913569 CHlqPKqkIdqwBCBUHreXbFAkCt 2024-05-25 1.5683842369495904 -1009 757881 AjcSyYMIMzS 2024-05-04 7.5674012939461255 -101 326164 QWLnalYNmYDt 2024-01-07 3.8159876011523854 -1010 427079 AlRUfmxfAuoLnPqUTvQVMtrS 2024-06-04 3.8087069699523313 -1011 252076 gHmFDhtytYzWETIxdpkpMUpnLd 2023-09-17 6.773606843056635 -1012 819615 rFfRHquexplDJvSeUK 2023-11-02 3.220639250504097 -1013 413456 uvNPelHXYjJKiOkwdNbmUkGzxiiqLo 2024-03-15 8.305048700108081 -1014 308042 vnzcsvHxnWFhvLwJkAtUqe 2024-06-15 1.5668867233009998 -1015 603837 VBEsRVGyhRNWQeKzDaBnJHmFDnXAOU 2024-08-17 3.8287482122289007 -1016 912679 eEjldPhxojSjTnE 2024-01-09 1.3717891874157961 -1017 630392 TcczYHXbwaCYzFSfXJlhsFjN 2023-10-07 4.733337480058437 +1 694832 buHDwfGeNHfpRFdNaogneddi 2024-02-09 4.8995886 +10 218729 goZsLvvWFOIjlzSAitC 2024-06-10 4.1377325 +100 813423 zICskqgcdPc 2024-03-23 8.4865294 +1000 612650 RzOXeYpKOmuJOogUyeIEDNDmvq 2023-12-05 7.8741751 +1001 29486 WoUAFJFuJNnwyqMnoDhX 2024-03-11 9.7582445 +1002 445363 OdTEeeWtxfcRwx 2024-08-01 0.39349455 +1003 707035 JAYnKxusVpGzYueACf 2023-11-14 5.37711 +1004 227858 JIFyjKzmbjkt 2024-03-24 5.7480378 +1005 539305 PlruLkSUSXZgaHafFriklrhCi 2023-11-08 4.1226354 +1006 145518 KCwqEcSCGuXrHerwn 2024-06-22 8.48229 +1007 939028 KzXhEMelsKVLbDMsEKh 2024-01-01 8.14445 +1008 913569 CHlqPKqkIdqwBCBUHreXbFAkCt 2024-05-25 1.5683843 +1009 757881 AjcSyYMIMzS 2024-05-04 7.5674014 +101 326164 QWLnalYNmYDt 2024-01-07 3.8159876 +1010 427079 AlRUfmxfAuoLnPqUTvQVMtrS 2024-06-04 3.808707 +1011 252076 gHmFDhtytYzWETIxdpkpMUpnLd 2023-09-17 6.7736068 +1012 819615 rFfRHquexplDJvSeUK 2023-11-02 3.2206392 +1013 413456 uvNPelHXYjJKiOkwdNbmUkGzxiiqLo 2024-03-15 8.3050489 +1014 308042 vnzcsvHxnWFhvLwJkAtUqe 2024-06-15 1.5668868 +1015 603837 VBEsRVGyhRNWQeKzDaBnJHmFDnXAOU 2024-08-17 3.8287482 +1016 912679 eEjldPhxojSjTnE 2024-01-09 1.3717892 +1017 630392 TcczYHXbwaCYzFSfXJlhsFjN 2023-10-07 4.7333374 -- !snappy_2 -- diff --git a/regression-test/data/json_p0/test_json_double.csv b/regression-test/data/json_p0/test_json_double.csv new file mode 100644 index 00000000000000..e928633659bc10 --- /dev/null +++ b/regression-test/data/json_p0/test_json_double.csv @@ -0,0 +1,2 @@ +2 {"rebookProfit":3.729672759600005773616970827788463793694972991943359375} +3 3.729672759600005773616970827788463793694972991943359375 \ No newline at end of file diff --git a/regression-test/data/json_p0/test_json_load_double.out b/regression-test/data/json_p0/test_json_load_double.out new file mode 100644 index 00000000000000..621c3a0910e26e --- /dev/null +++ b/regression-test/data/json_p0/test_json_load_double.out @@ -0,0 +1,11 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_select_src -- +3.72967275960001 +\N + +-- !sql_select_dst -- +1 3.72967275960001 +1 {"rebookProfit":3.72967275960001} +2 {"rebookProfit":3.72967275960001} +3 3.72967275960001 + diff --git a/regression-test/data/mv_p0/unique/unique_rewrite.out b/regression-test/data/mv_p0/unique_rewrite/unique_rewrite.out similarity index 100% rename from regression-test/data/mv_p0/unique/unique_rewrite.out rename to regression-test/data/mv_p0/unique_rewrite/unique_rewrite.out diff --git a/regression-test/data/nereids_p0/ddl/alter/test_nereids_alter_database_set_quota.out b/regression-test/data/nereids_p0/ddl/alter/test_nereids_alter_database_set_quota.out new file mode 100644 index 00000000000000..d353b2ca3a3c41 --- /dev/null +++ b/regression-test/data/nereids_p0/ddl/alter/test_nereids_alter_database_set_quota.out @@ -0,0 +1,31 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !show_data_sql -- +Total 0.000 0 0.000 +Quota 1024.000 TB 1073741824 +Left 1024.000 TB 1073741824 + +-- !show_data_sql_100m -- +Total 0.000 0 0.000 +Quota 100.000 MB 1073741824 +Left 100.000 MB 1073741824 + +-- !show_data_sql_1024g -- +Total 0.000 0 0.000 +Quota 1024.000 GB 1073741824 +Left 1024.000 GB 1073741824 + +-- !show_data_sql_100t -- +Total 0.000 0 0.000 +Quota 100.000 TB 1073741824 +Left 100.000 TB 1073741824 + +-- !show_data_sql_10t -- +Total 0.000 0 0.000 +Quota 10.000 TB 1073741824 +Left 10.000 TB 1073741824 + +-- !show_data_sql_replica_num -- +Total 0.000 0 0.000 +Quota 10.000 TB 102400 +Left 10.000 TB 102400 + diff --git a/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot4.out b/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot4.out index 5b500067986366..264a653fd3a3d0 100644 --- a/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot4.out +++ b/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot4.out @@ -1,14 +1,22 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select_star -- -4 -4 -4 d +-4 -4 -4 d +-4 -4 -4 d +1 1 1 a +1 1 1 a 1 1 1 a 2 2 2 b +2 2 2 b +2 2 2 b +3 -3 \N c +3 -3 \N c 3 -3 \N c 3 -3 \N c -- !select_mv -- --3 1 -2 7 -3 9 +-3 3 +2 21 +3 27 4 \N diff --git a/regression-test/data/query_p0/show/test_show_create_materialized_view.out b/regression-test/data/query_p0/show/test_show_create_materialized_view.out index 040ebf56f29fc5..1d874bb14d3c5a 100644 --- a/regression-test/data/query_p0/show/test_show_create_materialized_view.out +++ b/regression-test/data/query_p0/show/test_show_create_materialized_view.out @@ -1,4 +1,4 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !cmd -- -table_for_mv_test mv_show_create_materialized_view CREATE MATERIALIZED VIEW mv_show_create_materialized_view AS\n SELECT id, name, SUM(value) AS total_value\n FROM table_for_mv_test\n GROUP BY id, name;\n +table_for_mv_test mv_show_create_materialized_view \n CREATE MATERIALIZED VIEW mv_show_create_materialized_view \n AS \n SELECT id, name, SUM(value) AS total_value\n FROM table_for_mv_test\n GROUP BY id, name\n diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy index 97b6d038fa701c..b0a811a2e2fdaa 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy @@ -367,10 +367,12 @@ class StreamLoadAction implements SuiteAction { if (time > 0) { long elapsed = endTime - startTime - if (elapsed > time) { - log.info("Stream load consums more time than expected, elapsed ${elapsed} ms, expect ${time} ms") - } else { - log.info("Stream load consums time elapsed ${elapsed} ms, expect ${time} ms") + try { + // stream load may cost more time than expected in regression test, because of case run in parallel. + // So we allow stream load cost more time, use 3 * time as threshold. + Assert.assertTrue("Stream load Expect elapsed <= 3 * ${time}, but meet ${elapsed}", elapsed <= 3 * time) + } catch (Throwable t) { + throw new IllegalStateException("Stream load Expect elapsed <= 3 * ${time}, but meet ${elapsed}") } } } diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index 5cbed97829c990..155cb5868bce59 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -742,10 +742,16 @@ class Suite implements GroovyInterceptable { return result } + // Should use create_sync_mv, this method only check the sync mv in current db + // If has multi sync mv in db, may make mistake + @Deprecated void createMV(String sql) { (new CreateMVAction(context, sql)).run() } + // Should use create_sync_mv, this method only check the sync mv in current db + // If has multi sync mv in db, may make mistake + @Deprecated void createMV(String sql, String expection) { (new CreateMVAction(context, sql, expection)).run() } @@ -1475,80 +1481,101 @@ class Suite implements GroovyInterceptable { return debugPoint } - void waitingMTMVTaskFinishedByMvName(String mvName) { + def waitingMTMVTaskFinishedByMvName = { mvName, dbName = context.dbName -> Thread.sleep(2000); - String showTasks = "select TaskId,JobId,JobName,MvId,Status,MvName,MvDatabaseName,ErrorMsg from tasks('type'='mv') where MvName = '${mvName}' order by CreateTime ASC" + String showTasks = "select TaskId,JobId,JobName,MvId,Status,MvName,MvDatabaseName,ErrorMsg from tasks('type'='mv') where MvDatabaseName = '${dbName}' and MvName = '${mvName}' order by CreateTime DESC LIMIT 1" String status = "NULL" List> result long startTime = System.currentTimeMillis() long timeoutTimestamp = startTime + 5 * 60 * 1000 // 5 min - do { + List toCheckTaskRow = new ArrayList<>(); + while (timeoutTimestamp > System.currentTimeMillis() && (status != "SUCCESS")) { result = sql(showTasks) - logger.info("result: " + result.toString()) - if (!result.isEmpty()) { - status = result.last().get(4) - } + logger.info("current db is " + dbName + ", showTasks is " + showTasks) + if (result.isEmpty()) { + logger.info("waitingMTMVTaskFinishedByMvName toCheckTaskRow is empty") + Thread.sleep(1000); + continue; + } + toCheckTaskRow = result.get(0); + status = toCheckTaskRow.get(4) logger.info("The state of ${showTasks} is ${status}") Thread.sleep(1000); - } while (timeoutTimestamp > System.currentTimeMillis() && (status == 'PENDING' || status == 'RUNNING' || status == 'NULL')) + } if (status != "SUCCESS") { logger.info("status is not success") } Assert.assertEquals("SUCCESS", status) def show_tables = sql """ - show tables from ${result.last().get(6)}; + show tables from ${toCheckTaskRow.get(6)}; """ - def db_id = getDbId(result.last().get(6)) - def table_id = getTableId(result.last().get(6), mvName) + def db_id = getDbId(toCheckTaskRow.get(6)) + def table_id = getTableId(toCheckTaskRow.get(6), mvName) logger.info("waitingMTMVTaskFinished analyze mv name is " + mvName - + ", db name is " + result.last().get(6) + + ", db name is " + toCheckTaskRow.get(6) + ", show_tables are " + show_tables + ", db_id is " + db_id + ", table_id " + table_id) - sql "analyze table ${result.last().get(6)}.${mvName} with sync;" + sql "analyze table ${toCheckTaskRow.get(6)}.${mvName} with sync;" } - void waitingMTMVTaskFinishedByMvNameAllowCancel(String mvName) { + def waitingMTMVTaskFinishedByMvNameAllowCancel = {mvName, dbName = context.dbName -> Thread.sleep(2000); - String showTasks = "select TaskId,JobId,JobName,MvId,Status,MvName,MvDatabaseName,ErrorMsg from tasks('type'='mv') where MvName = '${mvName}' order by CreateTime ASC" + String showTasks = "select TaskId,JobId,JobName,MvId,Status,MvName,MvDatabaseName,ErrorMsg from tasks('type'='mv') where MvDatabaseName = '${dbName}' and MvName = '${mvName}' order by CreateTime DESC LIMIT 1" + String status = "NULL" List> result long startTime = System.currentTimeMillis() long timeoutTimestamp = startTime + 5 * 60 * 1000 // 5 min - do { + List toCheckTaskRow = new ArrayList<>(); + while (timeoutTimestamp > System.currentTimeMillis() && (status != "SUCCESS")) { result = sql(showTasks) - logger.info("result: " + result.toString()) - if (!result.isEmpty()) { - status = result.last().get(4) - } + logger.info("current db is " + dbName + ", showTasks result: " + result.toString()) + if (result.isEmpty()) { + logger.info("waitingMTMVTaskFinishedByMvName toCheckTaskRow is empty") + Thread.sleep(1000); + continue; + } + toCheckTaskRow = result.get(0) + status = toCheckTaskRow.get(4) logger.info("The state of ${showTasks} is ${status}") Thread.sleep(1000); - } while (timeoutTimestamp > System.currentTimeMillis() && (status == 'PENDING' || status == 'RUNNING' || status == 'NULL' || status == 'CANCELED')) + } if (status != "SUCCESS") { logger.info("status is not success") assertTrue(result.toString().contains("same table")) } // Need to analyze materialized view for cbo to choose the materialized view accurately - logger.info("waitingMTMVTaskFinished analyze mv name is " + result.last().get(5)) - sql "analyze table ${result.last().get(6)}.${mvName} with sync;" + logger.info("waitingMTMVTaskFinished analyze mv name is " + toCheckTaskRow.get(5)) + sql "analyze table ${toCheckTaskRow.get(6)}.${mvName} with sync;" } - void waitingMVTaskFinishedByMvName(String dbName, String tableName) { + void waitingMVTaskFinishedByMvName(String dbName, String tableName, String indexName) { Thread.sleep(2000) - String showTasks = "SHOW ALTER TABLE MATERIALIZED VIEW from ${dbName} where TableName='${tableName}' ORDER BY CreateTime ASC" + String showTasks = "SHOW ALTER TABLE MATERIALIZED VIEW from ${dbName} where TableName='${tableName}' ORDER BY CreateTime DESC" String status = "NULL" List> result long startTime = System.currentTimeMillis() long timeoutTimestamp = startTime + 5 * 60 * 1000 // 5 min - do { + List toCheckTaskRow = new ArrayList<>(); + while (timeoutTimestamp > System.currentTimeMillis() && (status != 'FINISHED')) { result = sql(showTasks) - logger.info("result: " + result.toString()) - if (!result.isEmpty()) { - status = result.last().get(8) + logger.info("crrent db is " + dbName + ", showTasks result: " + result.toString()) + // just consider current db + for (List taskRow : result) { + if (taskRow.get(5).equals(indexName)) { + toCheckTaskRow = taskRow; + } + } + if (toCheckTaskRow.isEmpty()) { + logger.info("waitingMVTaskFinishedByMvName toCheckTaskRow is empty") + Thread.sleep(1000); + continue; } + status = toCheckTaskRow.get(8) logger.info("The state of ${showTasks} is ${status}") Thread.sleep(1000); - } while (timeoutTimestamp > System.currentTimeMillis() && (status != 'FINISHED')) + } if (status != "FINISHED") { logger.info("status is not success") } @@ -1925,6 +1952,15 @@ class Suite implements GroovyInterceptable { return isReady } + def create_sync_mv = { db, table_name, mv_name, mv_sql -> + sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name} ON ${table_name};""" + sql""" + CREATE MATERIALIZED VIEW ${mv_name} + AS ${mv_sql} + """ + waitingMVTaskFinishedByMvName(db, table_name, mv_name) + } + def create_async_mv = { db, mv_name, mv_sql -> sql """DROP MATERIALIZED VIEW IF EXISTS ${db}.${mv_name}""" diff --git a/regression-test/suites/account_p0/test_nereids_authentication.groovy b/regression-test/suites/account_p0/test_nereids_authentication.groovy index c2ee5f12e8cb1d..4b8c644495c039 100644 --- a/regression-test/suites/account_p0/test_nereids_authentication.groovy +++ b/regression-test/suites/account_p0/test_nereids_authentication.groovy @@ -49,7 +49,7 @@ suite("test_nereids_authentication", "query") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } def tokens = context.config.jdbcUrl.split('/') diff --git a/regression-test/suites/account_p0/test_nereids_row_policy.groovy b/regression-test/suites/account_p0/test_nereids_row_policy.groovy index 38426dc09e6ef4..4463a41c90b4d8 100644 --- a/regression-test/suites/account_p0/test_nereids_row_policy.groovy +++ b/regression-test/suites/account_p0/test_nereids_row_policy.groovy @@ -28,7 +28,7 @@ suite("test_nereids_row_policy") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } def assertQueryResult = { size -> @@ -88,7 +88,7 @@ suite("test_nereids_row_policy") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } dropPolciy "policy0" diff --git a/regression-test/suites/auth_call/test_account_management_grant_auth.groovy b/regression-test/suites/auth_call/test_account_management_grant_auth.groovy index eff62e64f88e8a..a1e1f68916f90a 100644 --- a/regression-test/suites/auth_call/test_account_management_grant_auth.groovy +++ b/regression-test/suites/auth_call/test_account_management_grant_auth.groovy @@ -31,7 +31,7 @@ suite("test_account_management_grant_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_account_management_role_auth.groovy b/regression-test/suites/auth_call/test_account_management_role_auth.groovy index c3b278100815f8..dab9596b8c2ae5 100644 --- a/regression-test/suites/auth_call/test_account_management_role_auth.groovy +++ b/regression-test/suites/auth_call/test_account_management_role_auth.groovy @@ -32,7 +32,7 @@ suite("test_account_management_role_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_account_management_user_auth.groovy b/regression-test/suites/auth_call/test_account_management_user_auth.groovy index fe6d6805f47f9b..9cbb489615b41e 100644 --- a/regression-test/suites/auth_call/test_account_management_user_auth.groovy +++ b/regression-test/suites/auth_call/test_account_management_user_auth.groovy @@ -30,7 +30,7 @@ suite("test_account_management_user_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_assistant_command_auth.groovy b/regression-test/suites/auth_call/test_assistant_command_auth.groovy index 1b47e566ff31e8..ba539f129d7bc9 100644 --- a/regression-test/suites/auth_call/test_assistant_command_auth.groovy +++ b/regression-test/suites/auth_call/test_assistant_command_auth.groovy @@ -31,7 +31,7 @@ suite("test_assistant_command_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_cluster_management_auth.groovy b/regression-test/suites/auth_call/test_cluster_management_auth.groovy index 2061b9dbca773a..f769f29c967abc 100644 --- a/regression-test/suites/auth_call/test_cluster_management_auth.groovy +++ b/regression-test/suites/auth_call/test_cluster_management_auth.groovy @@ -66,7 +66,7 @@ suite ("test_cluster_management_auth","nonConcurrent,p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_database_management_auth.groovy b/regression-test/suites/auth_call/test_database_management_auth.groovy index fb643d9ee089e2..71f1902299654b 100644 --- a/regression-test/suites/auth_call/test_database_management_auth.groovy +++ b/regression-test/suites/auth_call/test_database_management_auth.groovy @@ -29,7 +29,7 @@ suite("test_database_management_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; error_in_cloud = "Unsupported" } diff --git a/regression-test/suites/auth_call/test_ddl_catalog_auth.groovy b/regression-test/suites/auth_call/test_ddl_catalog_auth.groovy index 80f71a2e16f94b..8d6b31d35c9217 100644 --- a/regression-test/suites/auth_call/test_ddl_catalog_auth.groovy +++ b/regression-test/suites/auth_call/test_ddl_catalog_auth.groovy @@ -29,7 +29,7 @@ suite("test_ddl_catalog_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } sql """create catalog if not exists ${catalogNameOther} properties ( diff --git a/regression-test/suites/auth_call/test_ddl_database_auth.groovy b/regression-test/suites/auth_call/test_ddl_database_auth.groovy index 2821a375ddb559..bdcf5ff7c39be7 100644 --- a/regression-test/suites/auth_call/test_ddl_database_auth.groovy +++ b/regression-test/suites/auth_call/test_ddl_database_auth.groovy @@ -30,7 +30,7 @@ suite("test_ddl_database_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_ddl_encryptkey_auth.groovy b/regression-test/suites/auth_call/test_ddl_encryptkey_auth.groovy index 0749bdab71780a..dfa469bfce0028 100644 --- a/regression-test/suites/auth_call/test_ddl_encryptkey_auth.groovy +++ b/regression-test/suites/auth_call/test_ddl_encryptkey_auth.groovy @@ -28,7 +28,7 @@ suite("test_ddl_encryptkey_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_ddl_file_auth.groovy b/regression-test/suites/auth_call/test_ddl_file_auth.groovy index e32c26ecc220d0..35ac9f2632e5cc 100644 --- a/regression-test/suites/auth_call/test_ddl_file_auth.groovy +++ b/regression-test/suites/auth_call/test_ddl_file_auth.groovy @@ -28,7 +28,7 @@ suite("test_ddl_file_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_ddl_function_auth.groovy b/regression-test/suites/auth_call/test_ddl_function_auth.groovy index 2fa524bf424bd0..a2e38f0eb6d6c8 100644 --- a/regression-test/suites/auth_call/test_ddl_function_auth.groovy +++ b/regression-test/suites/auth_call/test_ddl_function_auth.groovy @@ -28,7 +28,7 @@ suite("test_ddl_function_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_ddl_index_auth.groovy b/regression-test/suites/auth_call/test_ddl_index_auth.groovy index 53ba3a0d4b3123..61a727923be553 100644 --- a/regression-test/suites/auth_call/test_ddl_index_auth.groovy +++ b/regression-test/suites/auth_call/test_ddl_index_auth.groovy @@ -58,7 +58,7 @@ suite("test_ddl_index_auth","p0,auth_call") { logger.info("cluster:" + clusters) assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } sql """create database ${dbName}""" sql """ diff --git a/regression-test/suites/auth_call/test_ddl_job_auth.groovy b/regression-test/suites/auth_call/test_ddl_job_auth.groovy index 45798191e48568..ef75802637757b 100644 --- a/regression-test/suites/auth_call/test_ddl_job_auth.groovy +++ b/regression-test/suites/auth_call/test_ddl_job_auth.groovy @@ -30,7 +30,7 @@ suite("test_ddl_job_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_ddl_mask_view_auth.groovy b/regression-test/suites/auth_call/test_ddl_mask_view_auth.groovy index 590e75781f19f2..4a8bc2787dc62c 100644 --- a/regression-test/suites/auth_call/test_ddl_mask_view_auth.groovy +++ b/regression-test/suites/auth_call/test_ddl_mask_view_auth.groovy @@ -29,7 +29,7 @@ suite("test_ddl_mask_view_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_ddl_mtmv_auth.groovy b/regression-test/suites/auth_call/test_ddl_mtmv_auth.groovy index 4db2177ee6ce18..160b6b840b019b 100644 --- a/regression-test/suites/auth_call/test_ddl_mtmv_auth.groovy +++ b/regression-test/suites/auth_call/test_ddl_mtmv_auth.groovy @@ -30,7 +30,7 @@ suite("test_ddl_mtmv_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_ddl_mv_auth.groovy b/regression-test/suites/auth_call/test_ddl_mv_auth.groovy index 4dbf54fdf0df72..4b4810604409fe 100644 --- a/regression-test/suites/auth_call/test_ddl_mv_auth.groovy +++ b/regression-test/suites/auth_call/test_ddl_mv_auth.groovy @@ -30,7 +30,7 @@ suite("test_ddl_mv_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") @@ -80,9 +80,9 @@ suite("test_ddl_mv_auth","p0,auth_call") { connect(user, "${pwd}", context.config.jdbcUrl) { sql """use ${dbName}""" sql """create materialized view ${mvName} as select username from ${dbName}.${tableName};""" - waitingMVTaskFinishedByMvName(dbName, tableName) + waitingMVTaskFinishedByMvName(dbName, tableName, mvName) sql """alter table ${dbName}.${tableName} add rollup ${rollupName}(username)""" - waitingMVTaskFinishedByMvName(dbName, tableName) + waitingMVTaskFinishedByMvName(dbName, tableName, rollupName) def mv_res = sql """desc ${dbName}.${tableName} all;""" logger.info("mv_res: " + mv_res) diff --git a/regression-test/suites/auth_call/test_ddl_part_table_auth.groovy b/regression-test/suites/auth_call/test_ddl_part_table_auth.groovy index 34e4766e19e5e2..5217dc2d051860 100644 --- a/regression-test/suites/auth_call/test_ddl_part_table_auth.groovy +++ b/regression-test/suites/auth_call/test_ddl_part_table_auth.groovy @@ -28,7 +28,7 @@ suite("test_ddl_part_table_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_ddl_row_policy_auth.groovy b/regression-test/suites/auth_call/test_ddl_row_policy_auth.groovy index af1e074f8d7d8c..f9ac109f7f377f 100644 --- a/regression-test/suites/auth_call/test_ddl_row_policy_auth.groovy +++ b/regression-test/suites/auth_call/test_ddl_row_policy_auth.groovy @@ -29,7 +29,7 @@ suite("test_ddl_row_policy_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_ddl_sql_block_rule_auth.groovy b/regression-test/suites/auth_call/test_ddl_sql_block_rule_auth.groovy index 3941897a5e9dd5..568ea9723d58c3 100644 --- a/regression-test/suites/auth_call/test_ddl_sql_block_rule_auth.groovy +++ b/regression-test/suites/auth_call/test_ddl_sql_block_rule_auth.groovy @@ -29,7 +29,7 @@ suite("test_ddl_sql_block_rule_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_ddl_table_auth.groovy b/regression-test/suites/auth_call/test_ddl_table_auth.groovy index c96aeb0d5e2cad..47ac4e07abc6f9 100644 --- a/regression-test/suites/auth_call/test_ddl_table_auth.groovy +++ b/regression-test/suites/auth_call/test_ddl_table_auth.groovy @@ -33,7 +33,7 @@ suite("test_ddl_table_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_ddl_view_auth.groovy b/regression-test/suites/auth_call/test_ddl_view_auth.groovy index 05f263ada20066..1a915acdb3cd6f 100644 --- a/regression-test/suites/auth_call/test_ddl_view_auth.groovy +++ b/regression-test/suites/auth_call/test_ddl_view_auth.groovy @@ -29,7 +29,7 @@ suite("test_ddl_view_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_dml_analyze_auth.groovy b/regression-test/suites/auth_call/test_dml_analyze_auth.groovy index 59706f140e69b1..8bc6a070d61524 100644 --- a/regression-test/suites/auth_call/test_dml_analyze_auth.groovy +++ b/regression-test/suites/auth_call/test_dml_analyze_auth.groovy @@ -30,7 +30,7 @@ suite("test_dml_analyze_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_dml_broker_load_auth.groovy b/regression-test/suites/auth_call/test_dml_broker_load_auth.groovy index 3ec26146699bce..0ea44241bfae2f 100644 --- a/regression-test/suites/auth_call/test_dml_broker_load_auth.groovy +++ b/regression-test/suites/auth_call/test_dml_broker_load_auth.groovy @@ -42,7 +42,7 @@ suite("test_dml_broker_load_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_dml_cancel_profile_auth.groovy b/regression-test/suites/auth_call/test_dml_cancel_profile_auth.groovy index f1aa5d0fa95105..9bcb95c2830d9f 100644 --- a/regression-test/suites/auth_call/test_dml_cancel_profile_auth.groovy +++ b/regression-test/suites/auth_call/test_dml_cancel_profile_auth.groovy @@ -28,7 +28,7 @@ suite("test_dml_cancel_profile_auth","p0,auth_call,nonConcurrent") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_dml_delete_table_auth.groovy b/regression-test/suites/auth_call/test_dml_delete_table_auth.groovy index d325250f2ce696..aec3ee4a9947e5 100644 --- a/regression-test/suites/auth_call/test_dml_delete_table_auth.groovy +++ b/regression-test/suites/auth_call/test_dml_delete_table_auth.groovy @@ -30,7 +30,7 @@ suite("test_dml_delete_table_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_dml_export_table_auth.groovy b/regression-test/suites/auth_call/test_dml_export_table_auth.groovy index 12812fdf0dde09..ee5674c940a35c 100644 --- a/regression-test/suites/auth_call/test_dml_export_table_auth.groovy +++ b/regression-test/suites/auth_call/test_dml_export_table_auth.groovy @@ -36,7 +36,7 @@ suite("test_dml_export_table_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_dml_insert_auth.groovy b/regression-test/suites/auth_call/test_dml_insert_auth.groovy index 6cfe66cb10c6d5..5b8a20e18f9f70 100644 --- a/regression-test/suites/auth_call/test_dml_insert_auth.groovy +++ b/regression-test/suites/auth_call/test_dml_insert_auth.groovy @@ -31,7 +31,7 @@ suite("test_dml_insert_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_dml_multi_routine_load_auth.groovy b/regression-test/suites/auth_call/test_dml_multi_routine_load_auth.groovy index 11fd6c43db4602..5b8ecef7e57eae 100644 --- a/regression-test/suites/auth_call/test_dml_multi_routine_load_auth.groovy +++ b/regression-test/suites/auth_call/test_dml_multi_routine_load_auth.groovy @@ -34,7 +34,7 @@ suite("test_dml_multi_routine_load_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_dml_mysql_load_auth.groovy b/regression-test/suites/auth_call/test_dml_mysql_load_auth.groovy index 3963fe9433ef45..97151a0d001e85 100644 --- a/regression-test/suites/auth_call/test_dml_mysql_load_auth.groovy +++ b/regression-test/suites/auth_call/test_dml_mysql_load_auth.groovy @@ -30,7 +30,7 @@ suite("test_dml_mysql_load_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_dml_outfile_auth.groovy b/regression-test/suites/auth_call/test_dml_outfile_auth.groovy index 7edf476883a93b..5f4096344b8624 100644 --- a/regression-test/suites/auth_call/test_dml_outfile_auth.groovy +++ b/regression-test/suites/auth_call/test_dml_outfile_auth.groovy @@ -34,7 +34,7 @@ suite("test_dml_outfile_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_dml_routine_load_auth.groovy b/regression-test/suites/auth_call/test_dml_routine_load_auth.groovy index db6698b01af194..025ac555356aa7 100644 --- a/regression-test/suites/auth_call/test_dml_routine_load_auth.groovy +++ b/regression-test/suites/auth_call/test_dml_routine_load_auth.groovy @@ -33,7 +33,7 @@ suite("test_dml_routine_load_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_dml_select_udf_auth.groovy b/regression-test/suites/auth_call/test_dml_select_udf_auth.groovy index b2f565fc02ef8a..6aa74784969e30 100644 --- a/regression-test/suites/auth_call/test_dml_select_udf_auth.groovy +++ b/regression-test/suites/auth_call/test_dml_select_udf_auth.groovy @@ -34,7 +34,7 @@ suite("test_dml_select_udf_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_dml_stream_load_auth.groovy b/regression-test/suites/auth_call/test_dml_stream_load_auth.groovy index 26ee5526f1ea69..02b0b59ff390f6 100644 --- a/regression-test/suites/auth_call/test_dml_stream_load_auth.groovy +++ b/regression-test/suites/auth_call/test_dml_stream_load_auth.groovy @@ -29,7 +29,7 @@ suite("test_dml_stream_load_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_dml_update_table_auth.groovy b/regression-test/suites/auth_call/test_dml_update_table_auth.groovy index 153dad5026a930..eb258df2df2800 100644 --- a/regression-test/suites/auth_call/test_dml_update_table_auth.groovy +++ b/regression-test/suites/auth_call/test_dml_update_table_auth.groovy @@ -30,7 +30,7 @@ suite("test_dml_update_table_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_grant_show_view_priv_auth.groovy b/regression-test/suites/auth_call/test_grant_show_view_priv_auth.groovy index 3a736fe4703816..c05da39ce2e06c 100644 --- a/regression-test/suites/auth_call/test_grant_show_view_priv_auth.groovy +++ b/regression-test/suites/auth_call/test_grant_show_view_priv_auth.groovy @@ -32,7 +32,7 @@ suite("test_grant_show_view_priv_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_hive_base_case_auth.groovy b/regression-test/suites/auth_call/test_hive_base_case_auth.groovy index 271015bec7c154..0fb0d422ebd2c0 100644 --- a/regression-test/suites/auth_call/test_hive_base_case_auth.groovy +++ b/regression-test/suites/auth_call/test_hive_base_case_auth.groovy @@ -29,7 +29,7 @@ suite("test_hive_base_case_auth", "p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } String enabled = context.config.otherConfigs.get("enableHiveTest") diff --git a/regression-test/suites/auth_call/test_show_backend_auth.groovy b/regression-test/suites/auth_call/test_show_backend_auth.groovy index adccf3637e6e2a..13e1fa4fbe480c 100644 --- a/regression-test/suites/auth_call/test_show_backend_auth.groovy +++ b/regression-test/suites/auth_call/test_show_backend_auth.groovy @@ -26,7 +26,7 @@ suite("test_show_backend_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_show_broker_auth.groovy b/regression-test/suites/auth_call/test_show_broker_auth.groovy index d8fa9bd10ebe01..a364ad5b33146d 100644 --- a/regression-test/suites/auth_call/test_show_broker_auth.groovy +++ b/regression-test/suites/auth_call/test_show_broker_auth.groovy @@ -26,7 +26,7 @@ suite("test_show_broker_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_show_charset_auth.groovy b/regression-test/suites/auth_call/test_show_charset_auth.groovy index 3ca23f7f6becf7..14991bc99f9e38 100644 --- a/regression-test/suites/auth_call/test_show_charset_auth.groovy +++ b/regression-test/suites/auth_call/test_show_charset_auth.groovy @@ -27,7 +27,7 @@ suite("test_show_no_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_show_convert_light_sc_auth.groovy b/regression-test/suites/auth_call/test_show_convert_light_sc_auth.groovy index d54862f1710845..6f8387d892925b 100644 --- a/regression-test/suites/auth_call/test_show_convert_light_sc_auth.groovy +++ b/regression-test/suites/auth_call/test_show_convert_light_sc_auth.groovy @@ -26,7 +26,7 @@ suite("test_show_convert_light_sc_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_show_create_table_auth.groovy b/regression-test/suites/auth_call/test_show_create_table_auth.groovy index 2a85ea42e6c8da..166179bae5f6be 100644 --- a/regression-test/suites/auth_call/test_show_create_table_auth.groovy +++ b/regression-test/suites/auth_call/test_show_create_table_auth.groovy @@ -28,7 +28,7 @@ suite("test_show_create_table_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_show_data_auth.groovy b/regression-test/suites/auth_call/test_show_data_auth.groovy index c6f3b6dd1536ba..951ba564d42fcf 100644 --- a/regression-test/suites/auth_call/test_show_data_auth.groovy +++ b/regression-test/suites/auth_call/test_show_data_auth.groovy @@ -29,7 +29,7 @@ suite("test_show_data_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_show_database_id_auth.groovy b/regression-test/suites/auth_call/test_show_database_id_auth.groovy index e30dc8d0db74a4..d9d131ee7793b4 100644 --- a/regression-test/suites/auth_call/test_show_database_id_auth.groovy +++ b/regression-test/suites/auth_call/test_show_database_id_auth.groovy @@ -28,7 +28,7 @@ suite("test_show_database_id_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_show_dynamic_table_auth.groovy b/regression-test/suites/auth_call/test_show_dynamic_table_auth.groovy index 727d705664311b..ad560fbf18f79f 100644 --- a/regression-test/suites/auth_call/test_show_dynamic_table_auth.groovy +++ b/regression-test/suites/auth_call/test_show_dynamic_table_auth.groovy @@ -28,7 +28,7 @@ suite("test_show_dynamic_table_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_show_frontend_auth.groovy b/regression-test/suites/auth_call/test_show_frontend_auth.groovy index f4a9bc15b074ec..87f8e9c7c09a68 100644 --- a/regression-test/suites/auth_call/test_show_frontend_auth.groovy +++ b/regression-test/suites/auth_call/test_show_frontend_auth.groovy @@ -26,7 +26,7 @@ suite("test_show_frontend_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_show_grant_auth.groovy b/regression-test/suites/auth_call/test_show_grant_auth.groovy index 1223faf6739d4a..efd2e84e38a9f7 100644 --- a/regression-test/suites/auth_call/test_show_grant_auth.groovy +++ b/regression-test/suites/auth_call/test_show_grant_auth.groovy @@ -26,7 +26,7 @@ suite("test_show_grant_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_show_proc_auth.groovy b/regression-test/suites/auth_call/test_show_proc_auth.groovy index 1daf8d97f703f2..4608c64ca67016 100644 --- a/regression-test/suites/auth_call/test_show_proc_auth.groovy +++ b/regression-test/suites/auth_call/test_show_proc_auth.groovy @@ -26,7 +26,7 @@ suite("test_show_proc_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_show_query_stats_auth.groovy b/regression-test/suites/auth_call/test_show_query_stats_auth.groovy index ba951a47465ee3..7552038a7432b2 100644 --- a/regression-test/suites/auth_call/test_show_query_stats_auth.groovy +++ b/regression-test/suites/auth_call/test_show_query_stats_auth.groovy @@ -26,7 +26,7 @@ suite("test_show_query_stats_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_show_tablet_auth.groovy b/regression-test/suites/auth_call/test_show_tablet_auth.groovy index 3ac938d8462228..eb73a7b906be11 100644 --- a/regression-test/suites/auth_call/test_show_tablet_auth.groovy +++ b/regression-test/suites/auth_call/test_show_tablet_auth.groovy @@ -28,7 +28,7 @@ suite("test_show_tablet_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_call/test_show_typecast_auth.groovy b/regression-test/suites/auth_call/test_show_typecast_auth.groovy index 8df4a2e2dc18c2..9e5d7186f01e17 100644 --- a/regression-test/suites/auth_call/test_show_typecast_auth.groovy +++ b/regression-test/suites/auth_call/test_show_typecast_auth.groovy @@ -26,7 +26,7 @@ suite("test_show_typecast_auth","p0,auth_call") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } try_sql("DROP USER ${user}") diff --git a/regression-test/suites/auth_p0/test_backends_auth.groovy b/regression-test/suites/auth_p0/test_backends_auth.groovy index 753ae837c776e9..db76b2740fccaf 100644 --- a/regression-test/suites/auth_p0/test_backends_auth.groovy +++ b/regression-test/suites/auth_p0/test_backends_auth.groovy @@ -29,7 +29,7 @@ suite("test_backends_auth","p0,auth") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } sql """grant select_priv on regression_test to ${user}""" diff --git a/regression-test/suites/auth_p0/test_catalogs_auth.groovy b/regression-test/suites/auth_p0/test_catalogs_auth.groovy index 96ebcef7cf81cb..1b67282d8fe206 100644 --- a/regression-test/suites/auth_p0/test_catalogs_auth.groovy +++ b/regression-test/suites/auth_p0/test_catalogs_auth.groovy @@ -36,7 +36,7 @@ suite("test_catalogs_auth","p0,auth") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } sql """grant select_priv on regression_test to ${user}""" diff --git a/regression-test/suites/auth_p0/test_frontends_auth.groovy b/regression-test/suites/auth_p0/test_frontends_auth.groovy index 21fff527518e2b..0ac96e5c653827 100644 --- a/regression-test/suites/auth_p0/test_frontends_auth.groovy +++ b/regression-test/suites/auth_p0/test_frontends_auth.groovy @@ -29,7 +29,7 @@ suite("test_frontends_auth","p0,auth") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } sql """grant select_priv on regression_test to ${user}""" diff --git a/regression-test/suites/auth_p0/test_frontends_disks_auth.groovy b/regression-test/suites/auth_p0/test_frontends_disks_auth.groovy index 3767fdde0a5e92..f46ead3256a52c 100644 --- a/regression-test/suites/auth_p0/test_frontends_disks_auth.groovy +++ b/regression-test/suites/auth_p0/test_frontends_disks_auth.groovy @@ -29,7 +29,7 @@ suite("test_frontends_disks_auth","p0,auth") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } sql """grant select_priv on regression_test to ${user}""" diff --git a/regression-test/suites/auth_p0/test_master_slave_consistency_auth.groovy b/regression-test/suites/auth_p0/test_master_slave_consistency_auth.groovy index 90228ebf3a5130..0fc6b3a4063142 100644 --- a/regression-test/suites/auth_p0/test_master_slave_consistency_auth.groovy +++ b/regression-test/suites/auth_p0/test_master_slave_consistency_auth.groovy @@ -110,7 +110,7 @@ suite ("test_follower_consistent_auth","p0,auth") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } logger.info("url_tmp1:" + url_tmp1) diff --git a/regression-test/suites/auth_p0/test_mtmv_auth.groovy b/regression-test/suites/auth_p0/test_mtmv_auth.groovy index 52ecbebb70b268..a190edaa0224da 100644 --- a/regression-test/suites/auth_p0/test_mtmv_auth.groovy +++ b/regression-test/suites/auth_p0/test_mtmv_auth.groovy @@ -58,7 +58,7 @@ suite("test_mtmv_auth","p0,auth") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } sql """grant select_priv on regression_test to ${user}""" diff --git a/regression-test/suites/auth_p0/test_partition_values_tvf_auth.groovy b/regression-test/suites/auth_p0/test_partition_values_tvf_auth.groovy index 3f0ae7ea8d524c..b9ce9f4364619e 100644 --- a/regression-test/suites/auth_p0/test_partition_values_tvf_auth.groovy +++ b/regression-test/suites/auth_p0/test_partition_values_tvf_auth.groovy @@ -44,7 +44,7 @@ suite("test_partition_values_tvf_auth","p0,auth") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } sql """grant select_priv on regression_test to ${user}""" diff --git a/regression-test/suites/auth_p0/test_partitions_auth.groovy b/regression-test/suites/auth_p0/test_partitions_auth.groovy index 0b769f11567845..1a398b84b4e84d 100644 --- a/regression-test/suites/auth_p0/test_partitions_auth.groovy +++ b/regression-test/suites/auth_p0/test_partitions_auth.groovy @@ -48,7 +48,7 @@ suite("test_partitions_auth","p0,auth") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } sql """grant select_priv on regression_test to ${user}""" diff --git a/regression-test/suites/auth_p0/test_query_tvf_auth.groovy b/regression-test/suites/auth_p0/test_query_tvf_auth.groovy index 746eb47ce5b870..6353ca142a9067 100644 --- a/regression-test/suites/auth_p0/test_query_tvf_auth.groovy +++ b/regression-test/suites/auth_p0/test_query_tvf_auth.groovy @@ -48,7 +48,7 @@ suite("test_query_tvf_auth", "p0,auth,external,external_docker") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${dorisuser}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${dorisuser}"""; } sql """grant select_priv on regression_test to ${dorisuser}""" diff --git a/regression-test/suites/auth_p0/test_select_column_auth.groovy b/regression-test/suites/auth_p0/test_select_column_auth.groovy index 36cc2a0a09cf1c..ba1511c77c0a47 100644 --- a/regression-test/suites/auth_p0/test_select_column_auth.groovy +++ b/regression-test/suites/auth_p0/test_select_column_auth.groovy @@ -37,7 +37,7 @@ suite("test_select_column_auth","p0,auth") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } sql """create database ${dbName}""" sql("""use ${dbName}""") @@ -54,7 +54,22 @@ suite("test_select_column_auth","p0,auth") { sql """create view ${dbName}.${mv_name} as select * from ${dbName}.${tableName};""" sql """alter table ${dbName}.${tableName} add rollup ${rollup_name}(username)""" - sleep(5 * 1000) + + for (int i = 0; i < 20; ++i) { + def r = sql_return_maparray """show alter table rollup where TableName='${tableName}' order by createtime desc limit 1""" + if (r.size() > 0) { + if ( r[0]["State"] == "FINISHED") { + break + } else if (r[0]["State"] == "CANCELLED") { + assertTrue(1==0) + } else { + sleep(1000) + } + } + sleep(1000) + } + sleep(1000) + createMV("""create materialized view ${mtmv_name} as select username from ${dbName}.${tableName}""") sleep(5 * 1000) sql """CREATE MATERIALIZED VIEW ${dbName}.${mtmv_name} @@ -69,7 +84,7 @@ suite("test_select_column_auth","p0,auth") { (3, "333"); """ sql """refresh MATERIALIZED VIEW ${dbName}.${mtmv_name} auto""" - waitingMTMVTaskFinishedByMvName(mtmv_name) + waitingMTMVTaskFinishedByMvName(mtmv_name, dbName) sql """grant select_priv on regression_test to ${user}""" diff --git a/regression-test/suites/auth_p0/test_select_count_auth.groovy b/regression-test/suites/auth_p0/test_select_count_auth.groovy index ccea1a4a580098..47a199aaca2291 100644 --- a/regression-test/suites/auth_p0/test_select_count_auth.groovy +++ b/regression-test/suites/auth_p0/test_select_count_auth.groovy @@ -29,7 +29,7 @@ suite("test_select_count_auth","p0,auth") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } sql """grant select_priv on regression_test to ${user}""" diff --git a/regression-test/suites/auth_p0/test_select_view_auth.groovy b/regression-test/suites/auth_p0/test_select_view_auth.groovy index 9a5020ea163ce6..6932fbbb58ded2 100644 --- a/regression-test/suites/auth_p0/test_select_view_auth.groovy +++ b/regression-test/suites/auth_p0/test_select_view_auth.groovy @@ -37,7 +37,7 @@ suite("test_select_view_auth","p0,auth") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } sql """create database ${dbName}""" sql("""use ${dbName}""") diff --git a/regression-test/suites/auth_p0/test_use_encryptkey_auth.groovy b/regression-test/suites/auth_p0/test_use_encryptkey_auth.groovy index 965bd4b3b162bb..258c1726ebfa62 100644 --- a/regression-test/suites/auth_p0/test_use_encryptkey_auth.groovy +++ b/regression-test/suites/auth_p0/test_use_encryptkey_auth.groovy @@ -32,7 +32,7 @@ suite("test_use_encryptkey_auth","p0,auth") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } sql """CREATE ENCRYPTKEY ${key} AS 'ABCD123456789'""" diff --git a/regression-test/suites/auth_up_down_p0/load.groovy b/regression-test/suites/auth_up_down_p0/load.groovy index 7ac11b627abe9e..734293185897e5 100644 --- a/regression-test/suites/auth_up_down_p0/load.groovy +++ b/regression-test/suites/auth_up_down_p0/load.groovy @@ -50,8 +50,8 @@ suite("test_upgrade_downgrade_prepare_auth","p0,auth,restart_fe") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user1}"""; - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user2}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user1}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user2}"""; } try_sql """drop table if exists ${dbName}.${tableName1}""" diff --git a/regression-test/suites/bloom_filter_p0/test_bloom_filter.groovy b/regression-test/suites/bloom_filter_p0/test_bloom_filter.groovy index ff8710c5998fac..9fdd88b283752e 100644 --- a/regression-test/suites/bloom_filter_p0/test_bloom_filter.groovy +++ b/regression-test/suites/bloom_filter_p0/test_bloom_filter.groovy @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -suite("test_bloom_filter") { +suite("test_bloom_filter","nonConcurrent") { // todo: test bloom filter, such alter table bloom filter, create table with bloom filter sql "SHOW ALTER TABLE COLUMN" diff --git a/regression-test/suites/check_before_quit/check_before_quit.groovy b/regression-test/suites/check_before_quit/check_before_quit.groovy index ad35ce3db7106b..0fc2c2621bf743 100644 --- a/regression-test/suites/check_before_quit/check_before_quit.groovy +++ b/regression-test/suites/check_before_quit/check_before_quit.groovy @@ -269,11 +269,13 @@ suite("check_before_quit", "nonConcurrent,p0") { sql "drop materialized view if exists ${tbl}" } else { sql "drop table if exists ${tbl}" + // only re create table, because the table which view depends may be dropped, + // so recreate view may fail + sql(createTableSql[0][1]) + def createTableSqlResult = sql "show create table ${tbl}" + logger.info("create table/view sql result info: ${createTableSqlResult}") + assertEquals(createTableSqlResult, createTableSql) } - sql(createTableSql[0][1]) - def createTableSqlResult = sql "show create table ${tbl}" - logger.info("create table/view sql result info: ${createTableSqlResult}") - assertEquals(createTableSqlResult, createTableSql) } } diff --git a/regression-test/suites/correctness_p0/test_mv_case/test_mv_case.groovy b/regression-test/suites/correctness_p0/test_mv_case/test_mv_case.groovy index 8e548eb27ea87c..eae3d1fb3591fe 100644 --- a/regression-test/suites/correctness_p0/test_mv_case/test_mv_case.groovy +++ b/regression-test/suites/correctness_p0/test_mv_case/test_mv_case.groovy @@ -16,6 +16,7 @@ // under the License. suite("test_mv_case") { + sql """drop table if exists test_table_aaa2;""" sql """CREATE TABLE `test_table_aaa2` ( `ordernum` varchar(65533) NOT NULL , @@ -29,7 +30,7 @@ suite("test_mv_case") { "replication_allocation" = "tag.location.default: 1" );""" sql """DROP MATERIALIZED VIEW IF EXISTS ods_zn_dnt_max1 ON test_table_aaa2;""" - createMV("""create materialized view ods_zn_dnt_max1 as + create_sync_mv(context.dbName, "test_table_aaa2", "ods_zn_dnt_max1", """ select ordernum,max(dnt) as dnt from test_table_aaa2 group by ordernum ORDER BY ordernum;""") @@ -92,7 +93,7 @@ suite("test_mv_case") { ) """ sql """insert into tb1 select id,map_agg(a, b) from(select 123 id,3 a,'5' b union all select 123 id, 6 a, '8' b) aa group by id""" - createMV ("""CREATE MATERIALIZED VIEW mv1 BUILD IMMEDIATE REFRESH COMPLETE ON SCHEDULE EVERY 10 MINUTE DUPLICATE KEY(info_id) DISTRIBUTED BY HASH(`info_id`) BUCKETS 2 PROPERTIES ( + sql"""CREATE MATERIALIZED VIEW mv1 BUILD IMMEDIATE REFRESH COMPLETE ON SCHEDULE EVERY 10 MINUTE DUPLICATE KEY(info_id) DISTRIBUTED BY HASH(`info_id`) BUCKETS 2 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "min_load_replica_num" = "-1", "is_being_synced" = "false", @@ -112,8 +113,9 @@ suite("test_mv_case") { cast(a.id as bigint) info_id, map_infos from - tb1 a;""") - createMV ("""CREATE MATERIALIZED VIEW mv2 BUILD IMMEDIATE REFRESH COMPLETE ON SCHEDULE EVERY 10 MINUTE DUPLICATE KEY(info_id) DISTRIBUTED BY HASH(`info_id`) BUCKETS 2 PROPERTIES ( + tb1 a;""" + waitingMTMVTaskFinishedByMvName("mv1") + sql """CREATE MATERIALIZED VIEW mv2 BUILD IMMEDIATE REFRESH COMPLETE ON SCHEDULE EVERY 10 MINUTE DUPLICATE KEY(info_id) DISTRIBUTED BY HASH(`info_id`) BUCKETS 2 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "min_load_replica_num" = "-1", "is_being_synced" = "false", @@ -132,6 +134,7 @@ suite("test_mv_case") { info_id, map_infos from - mv1 a;""") + mv1 a;""" + waitingMTMVTaskFinishedByMvName("mv2") qt_select_mv """ select * from mv2 """ } diff --git a/regression-test/suites/ddl_p0/create_view_nereids/create_view_use_mv.groovy b/regression-test/suites/ddl_p0/create_view_nereids/create_view_use_mv.groovy index 295b195aa58954..7b6069968f4a0a 100644 --- a/regression-test/suites/ddl_p0/create_view_nereids/create_view_use_mv.groovy +++ b/regression-test/suites/ddl_p0/create_view_nereids/create_view_use_mv.groovy @@ -48,8 +48,8 @@ suite("create_view_use_mv") { (3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'), (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17');""" - createMV(""" - CREATE MATERIALIZED VIEW t_mv_mv AS select + create_sync_mv(context.dbName, "orders", "t_mv_mv", """ + select o_orderkey, sum(o_totalprice) as sum_total, max(o_totalprice) as max_total, diff --git a/regression-test/suites/doc/admin-manual/data-admin/backup.md.groovy b/regression-test/suites/doc/admin-manual/data-admin/backup.md.groovy index 4c7f6406a2abdd..131c64cbb48fcd 100644 --- a/regression-test/suites/doc/admin-manual/data-admin/backup.md.groovy +++ b/regression-test/suites/doc/admin-manual/data-admin/backup.md.groovy @@ -17,13 +17,22 @@ import org.junit.jupiter.api.Assertions; -suite("docs/admin-manual/data-admin/backup.md", "p0,nonConcurrent") { +suite("docs/admin-manual/data-admin/backup.md", "backup_restore") { + if (isCloudMode()) { + logger.info("skip this case, because not supported in cloud mode") + return + } + if (!enableHdfs()) { + logger.info("skip this case, because hdfs is not enable") + return + } try { + def uuid = UUID.randomUUID().hashCode().abs() multi_sql """ - CREATE DATABASE IF NOT EXISTS example_db; - USE example_db; + CREATE DATABASE IF NOT EXISTS example_db${uuid}; + USE example_db${uuid}; DROP TABLE IF EXISTS example_tbl; - CREATE TABLE IF NOT EXISTS example_db.example_tbl( + CREATE TABLE IF NOT EXISTS example_db${uuid}.example_tbl( a INT ) PARTITION BY RANGE(a) ( PARTITION p1 VALUES LESS THAN (1), @@ -38,7 +47,6 @@ suite("docs/admin-manual/data-admin/backup.md", "p0,nonConcurrent") { INSERT INTO example_tbl2 SELECT * FROM example_tbl; """ - def uuid = UUID.randomUUID().hashCode().abs() def syncer = getSyncer() /* CREATE REPOSITORY `example_repo` @@ -50,7 +58,7 @@ suite("docs/admin-manual/data-admin/backup.md", "p0,nonConcurrent") { "hadoop.username" = "hadoop" ); */ - syncer.createHdfsRepository("example_repo") + syncer.createHdfsRepository("example_repo${uuid}") /* CREATE REPOSITORY `s3_repo` @@ -64,28 +72,28 @@ suite("docs/admin-manual/data-admin/backup.md", "p0,nonConcurrent") { "AWS_REGION" = "xxx" ); */ - syncer.createS3Repository("s3_repo") + syncer.createS3Repository("s3_repo${uuid}") sql """ - BACKUP SNAPSHOT example_db.snapshot_label1${uuid} - TO example_repo + BACKUP SNAPSHOT example_db${uuid}.snapshot_label1${uuid} + TO example_repo${uuid} ON (example_tbl) PROPERTIES ("type" = "full"); """ - syncer.waitSnapshotFinish("example_db") + syncer.waitSnapshotFinish("example_db${uuid}") sql """ - BACKUP SNAPSHOT example_db.snapshot_label2${uuid} - TO example_repo + BACKUP SNAPSHOT example_db${uuid}.snapshot_label2${uuid} + TO example_repo${uuid} ON ( example_tbl PARTITION (p1,p2), example_tbl2 ); """ - syncer.waitSnapshotFinish("example_db") + syncer.waitSnapshotFinish("example_db${uuid}") sql """show BACKUP""" - sql """SHOW SNAPSHOT ON example_repo WHERE SNAPSHOT = "snapshot_label1${uuid}";""" + sql """SHOW SNAPSHOT ON example_repo${uuid} WHERE SNAPSHOT = "snapshot_label1${uuid}";""" } catch (Throwable t) { Assertions.fail("examples in docs/admin-manual/data-admin/backup.md failed to exec, please fix it", t) diff --git a/regression-test/suites/doc/admin-manual/data-admin/restore.md.groovy b/regression-test/suites/doc/admin-manual/data-admin/restore.md.groovy index f36d225854bae6..e0fb1d4ccfdba0 100644 --- a/regression-test/suites/doc/admin-manual/data-admin/restore.md.groovy +++ b/regression-test/suites/doc/admin-manual/data-admin/restore.md.groovy @@ -17,13 +17,18 @@ import org.junit.jupiter.api.Assertions; -suite("docs/admin-manual/data-admin/restore.md", "p0,nonConcurrent") { +suite("docs/admin-manual/data-admin/restore.md") { + if (isCloudMode()) { + logger.info("skip this case, because not supported in cloud mode") + return + } try { + def uuid = UUID.randomUUID().hashCode().abs() multi_sql """ - CREATE DATABASE IF NOT EXISTS example_db1; - USE example_db1; + CREATE DATABASE IF NOT EXISTS example_db${uuid}; + USE example_db${uuid}; DROP TABLE IF EXISTS backup_tbl; - CREATE TABLE IF NOT EXISTS example_db1.backup_tbl( + CREATE TABLE IF NOT EXISTS example_db${uuid}.backup_tbl( a INT ) PARTITION BY RANGE(a) ( PARTITION p1 VALUES LESS THAN (1), @@ -38,33 +43,32 @@ suite("docs/admin-manual/data-admin/restore.md", "p0,nonConcurrent") { INSERT INTO backup_tbl2 SELECT * FROM backup_tbl; """ - def uuid = UUID.randomUUID().hashCode().abs() def syncer = getSyncer() - syncer.createS3Repository("example_repo") + syncer.createS3Repository("example_repo${uuid}") sql """ - BACKUP SNAPSHOT example_db1.snapshot_1${uuid} - TO example_repo + BACKUP SNAPSHOT example_db${uuid}.snapshot_1${uuid} + TO example_repo${uuid} ON (backup_tbl) PROPERTIES ("type" = "full"); """ - syncer.waitSnapshotFinish("example_db1") + syncer.waitSnapshotFinish("example_db${uuid}") sql """ - BACKUP SNAPSHOT example_db1.snapshot_2${uuid} - TO example_repo + BACKUP SNAPSHOT example_db${uuid}.snapshot_2${uuid} + TO example_repo${uuid} ON (backup_tbl, backup_tbl2) PROPERTIES ("type" = "full"); """ - syncer.waitSnapshotFinish("example_db1") + syncer.waitSnapshotFinish("example_db${uuid}") multi_sql """ truncate table backup_tbl; truncate table backup_tbl2; """ - var timestamp = syncer.getSnapshotTimestamp("example_repo", "snapshot_1${uuid}") + var timestamp = syncer.getSnapshotTimestamp("example_repo${uuid}", "snapshot_1${uuid}") sql """ - RESTORE SNAPSHOT example_db1.`snapshot_1${uuid}` - FROM `example_repo` + RESTORE SNAPSHOT example_db${uuid}.`snapshot_1${uuid}` + FROM `example_repo${uuid}` ON ( `backup_tbl` ) PROPERTIES ( @@ -72,12 +76,12 @@ suite("docs/admin-manual/data-admin/restore.md", "p0,nonConcurrent") { "replication_num" = "1" ); """ - syncer.waitAllRestoreFinish("example_db1") + syncer.waitAllRestoreFinish("example_db${uuid}") - var timestamp2 = syncer.getSnapshotTimestamp("example_repo", "snapshot_2${uuid}") + var timestamp2 = syncer.getSnapshotTimestamp("example_repo${uuid}", "snapshot_2${uuid}") sql """ - RESTORE SNAPSHOT example_db1.`snapshot_2${uuid}` - FROM `example_repo` + RESTORE SNAPSHOT example_db${uuid}.`snapshot_2${uuid}` + FROM `example_repo${uuid}` ON ( `backup_tbl` PARTITION (`p1`, `p2`), @@ -89,7 +93,7 @@ suite("docs/admin-manual/data-admin/restore.md", "p0,nonConcurrent") { "replication_num" = "1" ); """ - syncer.waitAllRestoreFinish("example_db1") + syncer.waitAllRestoreFinish("example_db${uuid}") sql """SHOW RESTORE""" diff --git a/regression-test/suites/external_table_p0/hive/test_autoinc_broker_load.groovy b/regression-test/suites/external_table_p0/hive/test_autoinc_broker_load.groovy index cefbd4e69942d6..106263203fcd74 100644 --- a/regression-test/suites/external_table_p0/hive/test_autoinc_broker_load.groovy +++ b/regression-test/suites/external_table_p0/hive/test_autoinc_broker_load.groovy @@ -105,7 +105,7 @@ suite("test_autoinc_broker_load", "p0,external,hive,external_docker,external_doc load_from_hdfs("id, name, value", table, test_load_label, "auto_inc_with_null.csv", "csv") wait_for_load_result(test_load_label, table) sql "sync" - qt_sql "select * from ${table};" + qt_sql "select * from ${table} order by id;" sql """ insert into ${table} values(0, "Bob", 123), (2, "Tom", 323), (4, "Carter", 523);""" qt_sql "select * from ${table} order by id" sql "drop table if exists ${table};" diff --git a/regression-test/suites/external_table_p0/jdbc/test_jdbc_call.groovy b/regression-test/suites/external_table_p0/jdbc/test_jdbc_call.groovy index e9e00b7084fba0..7f4d7403c3d43a 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_jdbc_call.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_jdbc_call.groovy @@ -118,8 +118,8 @@ suite("test_jdbc_call", "p0,external,doris,external_docker,external_docker_doris def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user1}"""; - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user2}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user1}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user2}"""; } def result1 = connect("${user1}", "", context.config.jdbcUrl) { diff --git a/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy index 505c5208c9988c..e1612bcf884779 100644 --- a/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy +++ b/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy @@ -25,6 +25,7 @@ suite("test_single_hive_kerberos", "p0,external,kerberos,external_docker,externa println "Docker containers:" println output String enabled = context.config.otherConfigs.get("enableKerberosTest") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") if (enabled != null && enabled.equalsIgnoreCase("true")) { String hms_catalog_name = "test_single_hive_kerberos" sql """drop catalog if exists hms_kerberos;""" @@ -32,8 +33,8 @@ suite("test_single_hive_kerberos", "p0,external,kerberos,external_docker,externa CREATE CATALOG IF NOT EXISTS hms_kerberos PROPERTIES ( "type" = "hms", - "hive.metastore.uris" = "thrift://172.31.71.25:9083", - "fs.defaultFS" = "hdfs://172.31.71.25:8020", + "hive.metastore.uris" = "thrift://${externalEnvIp}:9583", + "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", "hadoop.security.authentication" = "kerberos", "hadoop.kerberos.principal"="presto-server/presto-master.docker.cluster@LABS.TERADATA.COM", "hadoop.kerberos.keytab" = "/keytabs/presto-server.keytab", @@ -42,7 +43,7 @@ suite("test_single_hive_kerberos", "p0,external,kerberos,external_docker,externa RULE:[2:\$1@\$0](.*@OTHERREALM.COM)s/@.*// DEFAULT", "hive.metastore.sasl.enabled " = "true", - "hive.metastore.kerberos.principal" = "hive/_HOST@LABS.TERADATA.COM" + "hive.metastore.kerberos.principal" = "hive/hadoop-master@LABS.TERADATA.COM" ); """ sql """ switch hms_kerberos """ @@ -56,8 +57,8 @@ suite("test_single_hive_kerberos", "p0,external,kerberos,external_docker,externa CREATE CATALOG IF NOT EXISTS hms_kerberos_hadoop_err1 PROPERTIES ( "type" = "hms", - "hive.metastore.uris" = "thrift://172.31.71.25:9083", - "fs.defaultFS" = "hdfs://172.31.71.25:8020", + "hive.metastore.uris" = "thrift://${externalEnvIp}:9583", + "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", "hadoop.security.authentication" = "kerberos", "hadoop.kerberos.principal"="presto-server/presto-master.docker.cluster@LABS.TERADATA.COM", "hadoop.kerberos.keytab" = "/keytabs/presto-server.keytab" @@ -79,8 +80,8 @@ suite("test_single_hive_kerberos", "p0,external,kerberos,external_docker,externa PROPERTIES ( "type" = "hms", "hive.metastore.sasl.enabled " = "true", - "hive.metastore.uris" = "thrift://172.31.71.25:9083", - "fs.defaultFS" = "hdfs://172.31.71.25:8020" + "hive.metastore.uris" = "thrift://${externalEnvIp}:9583", + "fs.defaultFS" = "hdfs://${externalEnvIp}:8520" ); """ sql """ switch hms_kerberos_hadoop_err2 """ @@ -95,8 +96,8 @@ suite("test_single_hive_kerberos", "p0,external,kerberos,external_docker,externa // CREATE CATALOG IF NOT EXISTS hms_keberos_ccache // PROPERTIES ( // "type" = "hms", - // "hive.metastore.uris" = "thrift://172.31.71.25:9083", - // "fs.defaultFS" = "hdfs://172.31.71.25:8020", + // "hive.metastore.uris" = "thrift://${externalEnvIp}:9583", + // "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", // "hadoop.security.authentication" = "kerberos", // "hadoop.kerberos.principal"="presto-server/presto-master.docker.cluster@LABS.TERADATA.COM", // "hadoop.kerberos.keytab" = "/keytabs/presto-server.keytab", diff --git a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy index 29b0cb2cd7e9c6..764078c62a6c82 100644 --- a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy +++ b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy @@ -29,6 +29,7 @@ suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_d println "Docker containers:" println output String enabled = context.config.otherConfigs.get("enableKerberosTest") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") if (enabled != null && enabled.equalsIgnoreCase("true")) { String hms_catalog_name = "test_two_hive_kerberos" sql """drop catalog if exists ${hms_catalog_name};""" @@ -36,14 +37,14 @@ suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_d CREATE CATALOG IF NOT EXISTS ${hms_catalog_name} PROPERTIES ( "type" = "hms", - "hive.metastore.uris" = "thrift://172.31.71.25:9083", - "fs.defaultFS" = "hdfs://172.31.71.25:8020", + "hive.metastore.uris" = "thrift://${externalEnvIp}:9583", + "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", "hadoop.kerberos.min.seconds.before.relogin" = "5", "hadoop.security.authentication" = "kerberos", "hadoop.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM", "hadoop.kerberos.keytab" = "/keytabs/hive-presto-master.keytab", "hive.metastore.sasl.enabled " = "true", - "hive.metastore.kerberos.principal" = "hive/_HOST@LABS.TERADATA.COM" + "hive.metastore.kerberos.principal" = "hive/hadoop-master@LABS.TERADATA.COM" ); """ @@ -52,14 +53,14 @@ suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_d CREATE CATALOG IF NOT EXISTS other_${hms_catalog_name} PROPERTIES ( "type" = "hms", - "hive.metastore.uris" = "thrift://172.31.71.26:9083", - "fs.defaultFS" = "hdfs://172.31.71.26:8020", + "hive.metastore.uris" = "thrift://${externalEnvIp}:9683", + "fs.defaultFS" = "hdfs://${externalEnvIp}:8620", "hadoop.kerberos.min.seconds.before.relogin" = "5", "hadoop.security.authentication" = "kerberos", "hadoop.kerberos.principal"="hive/presto-master.docker.cluster@OTHERREALM.COM", "hadoop.kerberos.keytab" = "/keytabs/other-hive-presto-master.keytab", "hive.metastore.sasl.enabled " = "true", - "hive.metastore.kerberos.principal" = "hive/_HOST@OTHERREALM.COM", + "hive.metastore.kerberos.principal" = "hive/hadoop-master-2@OTHERREALM.COM", "hadoop.security.auth_to_local" ="RULE:[2:\$1@\$0](.*@OTHERREALM.COM)s/@.*// RULE:[2:\$1@\$0](.*@OTHERLABS.TERADATA.COM)s/@.*// DEFAULT" diff --git a/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy b/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy index c8ed6c4cbbc888..92fe356dfd6c35 100644 --- a/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy +++ b/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy @@ -32,7 +32,7 @@ suite("test_conflict_name", "p0,external,doris,meta_names_mapping,external_docke def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${jdbcUser}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${jdbcUser}"""; } sql """grant all on *.*.* to ${jdbcUser}""" diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy index 63f18e358c71bf..91a56f7f317a69 100644 --- a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy @@ -32,7 +32,7 @@ suite("test_lower_case_meta_include", "p0,external,doris,external_docker,externa def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${jdbcUser}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${jdbcUser}"""; } sql """grant all on *.*.* to ${jdbcUser}""" diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy index 72e945ea8ffd59..8853d169a13bae 100644 --- a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy @@ -48,7 +48,7 @@ suite("test_lower_case_meta_show_and_select", "p0,external,doris,external_docker def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${jdbcUser}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${jdbcUser}"""; } sql """grant all on *.*.* to ${jdbcUser}""" diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_grant.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_grant.groovy index ab47a1e23bd63a..78baa9aa438452 100644 --- a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_grant.groovy +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_grant.groovy @@ -86,7 +86,7 @@ suite("test_lower_case_meta_with_lower_table_conf_auth", "p0,external,doris,exte def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } sql """grant select_priv on regression_test to ${user}""" @@ -134,7 +134,7 @@ suite("test_lower_case_meta_with_lower_table_conf_auth", "p0,external,doris,exte def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } sql """grant select_priv on regression_test to ${user}""" diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy index 42c6fd08203e4f..13750535628c44 100644 --- a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy @@ -48,7 +48,7 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${jdbcUser}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${jdbcUser}"""; } sql """grant all on *.*.* to ${jdbcUser}""" diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy index 4cc8593e459a6c..67081ed3afe12c 100644 --- a/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy @@ -32,7 +32,7 @@ suite("test_lower_case_mtmv", "p0,external,doris,external_docker,external_docker def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${jdbcUser}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${jdbcUser}"""; } sql """grant all on *.*.* to ${jdbcUser}""" diff --git a/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy b/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy index b677133015c996..c2073f9864cef6 100644 --- a/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy +++ b/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy @@ -32,7 +32,7 @@ suite("test_meta_cache_select_without_refresh", "p0,external,doris,external_dock def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${jdbcUser}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${jdbcUser}"""; } sql """grant all on *.*.* to ${jdbcUser}""" diff --git a/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy b/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy index 1ae22e1ba99020..a033b7e59f8691 100644 --- a/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy +++ b/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy @@ -32,7 +32,7 @@ suite("test_meta_names_mapping", "p0,external,doris,meta_names_mapping,external_ def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${jdbcUser}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${jdbcUser}"""; } sql """grant all on *.*.* to ${jdbcUser}""" diff --git a/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy b/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy index 5e0386330e3a51..a353e20d685074 100644 --- a/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy +++ b/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy @@ -53,7 +53,7 @@ suite("test_timing_refresh_catalog", "p0,external,doris,external_docker,external def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${jdbcUser}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${jdbcUser}"""; } sql """grant all on *.*.* to ${jdbcUser}""" diff --git a/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy b/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy index 053fad17785a2f..8be76c3e0f44a4 100644 --- a/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy +++ b/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy @@ -48,7 +48,7 @@ suite("test_upgrade_lower_case_catalog_prepare", "p0,external,doris,external_doc def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${jdbcUser}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${jdbcUser}"""; } sql """grant all on *.*.* to ${jdbcUser}""" diff --git a/regression-test/suites/external_table_p0/tvf/test_insert_from_tvf_with_common_user.groovy b/regression-test/suites/external_table_p0/tvf/test_insert_from_tvf_with_common_user.groovy index 103158c224c13f..f0ae51430fd4dd 100644 --- a/regression-test/suites/external_table_p0/tvf/test_insert_from_tvf_with_common_user.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_insert_from_tvf_with_common_user.groovy @@ -93,7 +93,7 @@ suite("test_insert_from_tvf_with_common_user", "p0,external,external_docker") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${common_user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${common_user}"""; } connect("${common_user}", '12345', context.config.jdbcUrl) { diff --git a/regression-test/suites/external_table_p0/tvf/test_s3_tvf_with_resource.groovy b/regression-test/suites/external_table_p0/tvf/test_s3_tvf_with_resource.groovy index 92091c18926bf9..1474ec0890b5fd 100644 --- a/regression-test/suites/external_table_p0/tvf/test_s3_tvf_with_resource.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_s3_tvf_with_resource.groovy @@ -203,7 +203,7 @@ suite("test_s3_tvf_with_resource", "p0,external,external_docker") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } // not have usage priv, can not select tvf with resource connect(user, "${pwd}", url) { diff --git a/regression-test/suites/external_table_p2/tvf/test_iceberg_meta.groovy b/regression-test/suites/external_table_p2/tvf/test_iceberg_meta.groovy index 557eaf5b061d70..3fc898b865fb17 100644 --- a/regression-test/suites/external_table_p2/tvf/test_iceberg_meta.groovy +++ b/regression-test/suites/external_table_p2/tvf/test_iceberg_meta.groovy @@ -63,7 +63,7 @@ suite("test_iceberg_meta", "p2,external,iceberg,external_remote,external_remote_ def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } sql """grant select_priv on regression_test to ${user}""" diff --git a/regression-test/suites/fault_injection_p0/test_multi_replica_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_multi_replica_fault_injection.groovy index d09983d52d0dc3..4e235daf97c20c 100644 --- a/regression-test/suites/fault_injection_p0/test_multi_replica_fault_injection.groovy +++ b/regression-test/suites/fault_injection_p0/test_multi_replica_fault_injection.groovy @@ -75,7 +75,7 @@ suite("test_multi_replica_fault_injection", "nonConcurrent") { file "baseall.txt" } - def load_with_injection = { injection, error_msg, success=false-> + def load_with_injection = { injection, error_msg, success=false, alt_error_msg=null-> try { sql "truncate table test" GetDebugPoint().enableDebugPointForAllBEs(injection) @@ -83,7 +83,8 @@ suite("test_multi_replica_fault_injection", "nonConcurrent") { assertTrue(success, String.format("Expected Exception '%s', actual success", error_msg)) } catch(Exception e) { logger.info(e.getMessage()) - assertTrue(e.getMessage().contains(error_msg), e.toString()) + boolean e_contains_alt_error_msg = (alt_error_msg != null && e.getMessage().contains(alt_error_msg)) + assertTrue(e.getMessage().contains(error_msg) || e_contains_alt_error_msg, e.toString()) } finally { GetDebugPoint().disableDebugPointForAllBEs(injection) } @@ -101,7 +102,7 @@ suite("test_multi_replica_fault_injection", "nonConcurrent") { // test one backend open failure load_with_injection("VTabletWriterV2._open_streams.skip_one_backend", "success", true) // test two backend open failure - load_with_injection("VTabletWriterV2._open_streams.skip_two_backends", "not enough streams 1/3") + load_with_injection("VTabletWriterV2._open_streams.skip_two_backends", "not enough streams 1/3", false, "succ replica num 1 < load required replica num 2") sql """ set enable_memtable_on_sink_node=false """ } } diff --git a/regression-test/suites/javaudf_p0/test_javaudf_auth.groovy b/regression-test/suites/javaudf_p0/test_javaudf_auth.groovy index 0729f14bb333f1..7f02c9218550ab 100644 --- a/regression-test/suites/javaudf_p0/test_javaudf_auth.groovy +++ b/regression-test/suites/javaudf_p0/test_javaudf_auth.groovy @@ -49,7 +49,7 @@ suite("test_javaudf_auth") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } sql """USE ${dbName}""" diff --git a/regression-test/suites/json_p0/test_json_load_double.groovy b/regression-test/suites/json_p0/test_json_load_double.groovy new file mode 100644 index 00000000000000..8c692e3e71da3d --- /dev/null +++ b/regression-test/suites/json_p0/test_json_load_double.groovy @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite("test_json_load_double", "p0") { + + def srcTable = "stringTable" + def dstTable = "jsonTable" + def dataFile = "test_json_double.csv" + + sql """ DROP TABLE IF EXISTS ${srcTable} """ + sql """ DROP TABLE IF EXISTS ${dstTable} """ + + sql """ + CREATE TABLE IF NOT EXISTS ${srcTable} ( + id INT not null, + v STRING not null + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num" = "1"); + """ + + sql """ + CREATE TABLE IF NOT EXISTS ${dstTable} ( + id INT not null, + j JSON not null + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num" = "1"); + """ + + sql """ + insert into ${srcTable} values(1,'{"rebookProfit":3.729672759600005773616970827788463793694972991943359375}'); + """ + + sql """ + insert into ${srcTable} values(1,'3.729672759600005773616970827788463793694972991943359375'); + """ + + sql """ insert into ${dstTable} select * from ${srcTable} """ + + // load the json data from csv file + streamLoad { + table dstTable + + file dataFile // import csv file + time 10000 // limit inflight 10s + set 'strict_mode', 'true' + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + + assertEquals("success", json.Status.toLowerCase()) + assertEquals(2, json.NumberTotalRows) + assertEquals(2, json.NumberLoadedRows) + assertTrue(json.LoadBytes > 0) + log.info("url: " + json.ErrorURL) + } + } + + qt_sql_select_src """ select jsonb_extract(v, '\$.rebookProfit') from ${srcTable} """ + qt_sql_select_dst """ select * from ${dstTable} """ + +} diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_with_user.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_with_user.groovy index 73cce57822fab3..73641f46d018c2 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_with_user.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_with_user.groovy @@ -65,7 +65,7 @@ suite("test_routine_load_with_user","p0") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; try { def storageVaults = (sql " SHOW STORAGE VAULT; ").stream().map(row -> row[0]).collect(Collectors.toSet()) diff --git a/regression-test/suites/load_p0/stream_load/test_stream_load.groovy b/regression-test/suites/load_p0/stream_load/test_stream_load.groovy index 54731a949584be..1bbf6033ef8ca0 100644 --- a/regression-test/suites/load_p0/stream_load/test_stream_load.groovy +++ b/regression-test/suites/load_p0/stream_load/test_stream_load.groovy @@ -1059,7 +1059,7 @@ suite("test_stream_load", "p0") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO common_user"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO common_user"""; } streamLoad { diff --git a/regression-test/suites/load_p0/stream_load/test_stream_load_move_memtable.groovy b/regression-test/suites/load_p0/stream_load/test_stream_load_move_memtable.groovy index bbd532a76259cc..1df859d03d144a 100644 --- a/regression-test/suites/load_p0/stream_load/test_stream_load_move_memtable.groovy +++ b/regression-test/suites/load_p0/stream_load/test_stream_load_move_memtable.groovy @@ -937,7 +937,7 @@ suite("test_stream_load_move_memtable", "p0") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ddd"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ddd"""; } streamLoad { diff --git a/regression-test/suites/manager/test_manager_interface_3.groovy b/regression-test/suites/manager/test_manager_interface_3.groovy index 22af98d5648ca7..b7baa8a869d94a 100644 --- a/regression-test/suites/manager/test_manager_interface_3.groovy +++ b/regression-test/suites/manager/test_manager_interface_3.groovy @@ -89,8 +89,8 @@ suite('test_manager_interface_3',"p0") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user1}"""; - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user2}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user1}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user2}"""; } connect(user1, "${pwd}", url) { @@ -412,7 +412,7 @@ suite('test_manager_interface_3',"p0") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } List> result = sql """ show resources """ @@ -609,7 +609,7 @@ suite('test_manager_interface_3',"p0") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } connect(user, "${pwd}", url) { diff --git a/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy b/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy index 36c0d3f120e109..8dd16ea571f1ec 100644 --- a/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy @@ -107,35 +107,35 @@ suite("test_iceberg_mtmv", "p0,external,iceberg,external_docker,external_docker_ sql """insert into ${catalog_name}.${icebergDb}.${icebergTable1} values ('2024-10-26 01:02:03', 1), ('2024-10-27 01:02:03', 2), ('2024-10-27 21:02:03', 3)""" sql """CREATE MATERIALIZED VIEW ${mvName1} BUILD DEFERRED REFRESH AUTO ON MANUAL partition by(`ts`) DISTRIBUTED BY RANDOM BUCKETS 2 PROPERTIES ('replication_num' = '1') as SELECT * FROM ${catalog_name}.${icebergDb}.${icebergTable1}""" sql """REFRESH MATERIALIZED VIEW ${mvName1} complete""" - waitingMTMVTaskFinishedByMvName(mvName1) + waitingMTMVTaskFinishedByMvName(mvName1, dbName) qt_test_ts_refresh1 "select * from ${mvName1} order by value" sql """insert into ${catalog_name}.${icebergDb}.${icebergTable1} values ('2024-10-26 21:02:03', 4)""" sql """REFRESH MATERIALIZED VIEW ${mvName1} auto""" - waitingMTMVTaskFinishedByMvName(mvName1) + waitingMTMVTaskFinishedByMvName(mvName1, dbName) qt_test_ts_refresh2 """select * from ${mvName1} order by value""" sql """insert into ${catalog_name}.${icebergDb}.${icebergTable1} values ('2024-10-26 01:22:03', 5), ('2024-10-27 01:12:03', 6);""" sql """REFRESH MATERIALIZED VIEW ${mvName1} partitions(p_20241026000000_20241027000000);""" - waitingMTMVTaskFinishedByMvName(mvName1) + waitingMTMVTaskFinishedByMvName(mvName1, dbName) qt_test_ts_refresh3 """select * from ${mvName1} order by value""" sql """REFRESH MATERIALIZED VIEW ${mvName1} auto""" - waitingMTMVTaskFinishedByMvName(mvName1) + waitingMTMVTaskFinishedByMvName(mvName1, dbName) qt_test_ts_refresh4 """select * from ${mvName1} order by value""" sql """insert into ${catalog_name}.${icebergDb}.${icebergTable1} values ('2024-10-28 01:22:03', 7);""" sql """REFRESH MATERIALIZED VIEW ${mvName1} partitions(p_20241026000000_20241027000000);""" - waitingMTMVTaskFinishedByMvName(mvName1) + waitingMTMVTaskFinishedByMvName(mvName1, dbName) qt_test_ts_refresh5 """select * from ${mvName1} order by value""" sql """REFRESH MATERIALIZED VIEW ${mvName1} auto""" - waitingMTMVTaskFinishedByMvName(mvName1) + waitingMTMVTaskFinishedByMvName(mvName1, dbName) qt_test_ts_refresh6 """select * from ${mvName1} order by value""" sql """insert into ${catalog_name}.${icebergDb}.${icebergTable1} values (null, 8);""" sql """REFRESH MATERIALIZED VIEW ${mvName1} auto""" - waitingMTMVTaskFinishedByMvName(mvName1) + waitingMTMVTaskFinishedByMvName(mvName1, dbName) qt_test_ts_refresh_null """select * from ${mvName1} order by value""" def showPartitionsResult = sql """show partitions from ${mvName1}""" @@ -176,25 +176,25 @@ suite("test_iceberg_mtmv", "p0,external,iceberg,external_docker,external_docker_ sql """insert into ${catalog_name}.${icebergDb}.${icebergTable2} values ('2024-08-26', 1), ('2024-09-17', 2), ('2024-09-27', 3);""" sql """CREATE MATERIALIZED VIEW ${mvName2} BUILD DEFERRED REFRESH AUTO ON MANUAL partition by(`d`) DISTRIBUTED BY RANDOM BUCKETS 2 PROPERTIES ('replication_num' = '1') as SELECT * FROM ${catalog_name}.${icebergDb}.${icebergTable2}""" sql """REFRESH MATERIALIZED VIEW ${mvName2} complete""" - waitingMTMVTaskFinishedByMvName(mvName2) + waitingMTMVTaskFinishedByMvName(mvName2, dbName) qt_test_d_refresh1 "select * from ${mvName2} order by value" sql """insert into ${catalog_name}.${icebergDb}.${icebergTable2} values ('2024-09-01', 4);""" sql """REFRESH MATERIALIZED VIEW ${mvName2} auto""" - waitingMTMVTaskFinishedByMvName(mvName2) + waitingMTMVTaskFinishedByMvName(mvName2, dbName) qt_test_d_refresh2 "select * from ${mvName2} order by value" sql """insert into ${catalog_name}.${icebergDb}.${icebergTable2} values ('2024-08-22', 5), ('2024-09-30', 6);""" sql """REFRESH MATERIALIZED VIEW ${mvName2} partitions(p_20240801_20240901);""" - waitingMTMVTaskFinishedByMvName(mvName2) + waitingMTMVTaskFinishedByMvName(mvName2, dbName) qt_test_d_refresh3 "select * from ${mvName2} order by value" sql """REFRESH MATERIALIZED VIEW ${mvName2} partitions(p_20240901_20241001);""" - waitingMTMVTaskFinishedByMvName(mvName2) + waitingMTMVTaskFinishedByMvName(mvName2, dbName) qt_test_d_refresh4 "select * from ${mvName2} order by value" sql """insert into ${catalog_name}.${icebergDb}.${icebergTable2} values ('2024-10-28', 7);""" sql """REFRESH MATERIALIZED VIEW ${mvName2} auto""" - waitingMTMVTaskFinishedByMvName(mvName2) + waitingMTMVTaskFinishedByMvName(mvName2, dbName) qt_test_d_refresh5 "select * from ${mvName2} order by value" showPartitionsResult = sql """show partitions from ${mvName2}""" @@ -240,7 +240,7 @@ suite("test_iceberg_mtmv", "p0,external,iceberg,external_docker,external_docker_ // refresh one partiton sql """REFRESH MATERIALIZED VIEW ${mvName} partitions(p_20240101000000_20240102000000);""" - waitingMTMVTaskFinishedByMvName(mvName) + waitingMTMVTaskFinishedByMvName(mvName, dbName) order_qt_refresh_one_partition "SELECT * FROM ${mvName} " def explainOnePartition = sql """ explain ${mvSql} """ logger.info("explainOnePartition: " + explainOnePartition.toString()) @@ -250,7 +250,7 @@ suite("test_iceberg_mtmv", "p0,external,iceberg,external_docker,external_docker_ //refresh auto sql """REFRESH MATERIALIZED VIEW ${mvName} auto""" - waitingMTMVTaskFinishedByMvName(mvName) + waitingMTMVTaskFinishedByMvName(mvName, dbName) order_qt_refresh_auto "SELECT * FROM ${mvName} " def explainAllPartition = sql """ explain ${mvSql}; """ logger.info("explainAllPartition: " + explainAllPartition.toString()) diff --git a/regression-test/suites/mtmv_p0/test_paimon_olap_rewrite_mtmv.groovy b/regression-test/suites/mtmv_p0/test_paimon_olap_rewrite_mtmv.groovy index a3ac1c048d30da..7a77cdc4590a4c 100644 --- a/regression-test/suites/mtmv_p0/test_paimon_olap_rewrite_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_paimon_olap_rewrite_mtmv.groovy @@ -87,6 +87,10 @@ suite("test_paimon_olap_rewrite_mtmv", "p0,external,mtmv,external_docker,externa def explainOnePartition = sql """ explain ${mvSql} """ logger.info("explainOnePartition: " + explainOnePartition.toString()) + + def explain_memo_plan = sql """ explain memo plan ${mvSql} """ + logger.info("explain_memo_plan: " + explain_memo_plan.toString()) + assertTrue(explainOnePartition.toString().contains("VUNION")) order_qt_refresh_one_partition_rewrite "${mvSql}" @@ -104,6 +108,10 @@ suite("test_paimon_olap_rewrite_mtmv", "p0,external,mtmv,external_docker,externa def explainAllPartition = sql """ explain ${mvSql}; """ logger.info("explainAllPartition: " + explainAllPartition.toString()) + + def explainMemoPlan = sql """ explain memo plan ${mvSql}; """ + logger.info("explainMemoPlan: " + explainMemoPlan.toString()) + assertTrue(explainAllPartition.toString().contains("VOlapScanNode")) order_qt_refresh_all_partition_rewrite "${mvSql}" diff --git a/regression-test/suites/mv_p0/no_await/no_await.groovy b/regression-test/suites/mv_p0/no_await/no_await.groovy index 866e4fdd5d3546..3eab03aa7e4adc 100644 --- a/regression-test/suites/mv_p0/no_await/no_await.groovy +++ b/regression-test/suites/mv_p0/no_await/no_await.groovy @@ -19,6 +19,8 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite ("no_await") { + String db = context.config.getDbNameByFile(context.file) + def tblName = "agg_have_dup_base_no_await" def waitDrop = { def try_times = 1000 @@ -58,92 +60,93 @@ suite ("no_await") { """ sql "insert into ${tblName} select e1, -4, -4, 'd' from (select 1 k1) as t lateral view explode_numbers(10000) tmp1 as e1;" // do not await - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") + sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" } diff --git a/regression-test/suites/mv_p0/unique/unique_rewrite.groovy b/regression-test/suites/mv_p0/unique_rewrite/unique_rewrite.groovy similarity index 100% rename from regression-test/suites/mv_p0/unique/unique_rewrite.groovy rename to regression-test/suites/mv_p0/unique_rewrite/unique_rewrite.groovy diff --git a/regression-test/suites/nereids_p0/authorization/column_authorization.groovy b/regression-test/suites/nereids_p0/authorization/column_authorization.groovy index c2e22f10c22678..eea353368c9684 100644 --- a/regression-test/suites/nereids_p0/authorization/column_authorization.groovy +++ b/regression-test/suites/nereids_p0/authorization/column_authorization.groovy @@ -49,7 +49,7 @@ suite("column_authorization") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user1}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user1}"""; } sql 'sync' diff --git a/regression-test/suites/nereids_p0/authorization/view_authorization.groovy b/regression-test/suites/nereids_p0/authorization/view_authorization.groovy index 51503c3cd2d737..fa7f56c1e09f0b 100644 --- a/regression-test/suites/nereids_p0/authorization/view_authorization.groovy +++ b/regression-test/suites/nereids_p0/authorization/view_authorization.groovy @@ -52,7 +52,7 @@ suite("view_authorization") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user1}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user1}"""; } sql 'sync' diff --git a/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy b/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy index e7fb5f3da6c435..765d1208426607 100644 --- a/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy +++ b/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy @@ -297,7 +297,7 @@ suite("parse_sql_from_sql_cache") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO test_cache_user1""" + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO test_cache_user1""" } createTestTable "test_use_plan_cache12" @@ -340,7 +340,7 @@ suite("parse_sql_from_sql_cache") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO test_cache_user2""" + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO test_cache_user2""" } createTestTable "test_use_plan_cache13" @@ -397,7 +397,7 @@ suite("parse_sql_from_sql_cache") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO test_cache_user3""" + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO test_cache_user3""" } createTestTable "test_use_plan_cache14" @@ -460,7 +460,7 @@ suite("parse_sql_from_sql_cache") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO test_cache_user4""" + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO test_cache_user4""" } sql "sync" diff --git a/regression-test/suites/nereids_p0/ddl/alter/test_nereids_alter_database_set_quota.groovy b/regression-test/suites/nereids_p0/ddl/alter/test_nereids_alter_database_set_quota.groovy new file mode 100644 index 00000000000000..6ff8256227f6a3 --- /dev/null +++ b/regression-test/suites/nereids_p0/ddl/alter/test_nereids_alter_database_set_quota.groovy @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_nereids_alter_database_set_quota") { + String quotaDb1= "quotaDb2" + + sql """DROP database IF EXISTS ${quotaDb1}""" + sql """create database ${quotaDb1}""" + sql """use ${quotaDb1}""" + qt_show_data_sql """show data;""" + checkNereidsExecute("ALTER DATABASE ${quotaDb1} SET DATA QUOTA 100M;"); + qt_show_data_sql_100m """show data""" + checkNereidsExecute("ALTER DATABASE ${quotaDb1} SET DATA QUOTA 1024G;"); + qt_show_data_sql_1024g """show data""" + checkNereidsExecute("ALTER DATABASE ${quotaDb1} SET DATA QUOTA 100T;"); + qt_show_data_sql_100t """show data""" + checkNereidsExecute("ALTER DATABASE ${quotaDb1} SET DATA QUOTA 10995116277760;"); + qt_show_data_sql_10t """show data""" + checkNereidsExecute("ALTER DATABASE ${quotaDb1} SET REPLICA QUOTA 102400;"); + qt_show_data_sql_replica_num """show data""" + checkNereidsExecute("ALTER DATABASE ${quotaDb1} SET TRANSACTION QUOTA 100000;"); + sql """drop database ${quotaDb1}""" +} diff --git a/regression-test/suites/nereids_p0/insert_into_table/insert_auth.groovy b/regression-test/suites/nereids_p0/insert_into_table/insert_auth.groovy index 4c9968746d13c1..d146885a7bf936 100644 --- a/regression-test/suites/nereids_p0/insert_into_table/insert_auth.groovy +++ b/regression-test/suites/nereids_p0/insert_into_table/insert_auth.groovy @@ -53,7 +53,7 @@ suite('nereids_insert_auth') { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } connect(user, "${pwd}", url) { diff --git a/regression-test/suites/nereids_rules_p0/mv/date_trunc/mv_with_date_trunc.groovy b/regression-test/suites/nereids_rules_p0/mv/date_trunc/mv_with_date_trunc.groovy index 58b2e7bbdd8ca3..e88f4132eacfba 100644 --- a/regression-test/suites/nereids_rules_p0/mv/date_trunc/mv_with_date_trunc.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/date_trunc/mv_with_date_trunc.groovy @@ -1447,7 +1447,7 @@ suite("mv_with_date_trunc") { logger.info("lineitem table stats: " + result) result = sql """show index stats lineitem lineitem""" logger.info("lineitem index stats: " + result) - mv_rewrite_success(query4_0, "mv4_0") + mv_rewrite_success(query4_0, "mv4_0", true, is_partition_statistics_ready(db, ["lineitem", "mv4_0"])) order_qt_query4_0_after "${query4_0}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv4_0""" diff --git a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy index 4870ec99e659c0..1972c2d505bfab 100644 --- a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy @@ -742,10 +742,12 @@ suite("nested_mtmv") { mv_rewrite_any_success(sql_2, [mv_1, mv_2]) compare_res(sql_2 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13") - mv_rewrite_any_success(sql_3, [mv_3, mv_4]) + // level 1 maybe use mv_1 and mv_2, this also meets expectation + mv_rewrite_any_success(sql_3, [mv_3, mv_4, mv_1, mv_2]) compare_res(sql_3 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13") - mv_rewrite_any_success(sql_4, [mv_3, mv_4]) + // level 1 maybe use mv_1 and mv_2, this also meets expectation + mv_rewrite_any_success(sql_4, [mv_3, mv_4, mv_1, mv_2]) compare_res(sql_4 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13") mv_rewrite_any_success(sql_5, [mv_3, mv_4, mv_5]) diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy index 7a49dbc4b37118..32e1684fdc9073 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy @@ -33,21 +33,29 @@ suite ("multi_slot4") { """ sql "insert into multi_slot4 select 1,1,1,'a';" + sql "insert into multi_slot4 select 1,1,1,'a';" + sql "insert into multi_slot4 select 1,1,1,'a';" + sql "insert into multi_slot4 select 2,2,2,'b';" + sql "insert into multi_slot4 select 2,2,2,'b';" sql "insert into multi_slot4 select 2,2,2,'b';" sql "insert into multi_slot4 select 3,-3,null,'c';" sql "insert into multi_slot4 select 3,-3,null,'c';" + sql "insert into multi_slot4 select 3,-3,null,'c';" + sql "insert into multi_slot4 select 3,-3,null,'c';" createMV ("create materialized view k1p2ap3ps as select k1+1,sum(abs(k2+2)+k3+3) from multi_slot4 group by k1+1;") sleep(3000) + sql "insert into multi_slot4 select -4,-4,-4,'d';" + sql "insert into multi_slot4 select -4,-4,-4,'d';" sql "insert into multi_slot4 select -4,-4,-4,'d';" sql "SET experimental_enable_nereids_planner=true" sql "SET enable_fallback_to_original_planner=false" sql "analyze table multi_slot4 with sync;" - sql """alter table multi_slot4 modify column k1 set stats ('row_count'='5');""" + sql """alter table multi_slot4 modify column k1 set stats ('row_count'='13');""" sql """set enable_stats=false;""" diff --git a/regression-test/suites/query_p0/show/test_show_create_materialized_view.groovy b/regression-test/suites/query_p0/show/test_show_create_materialized_view.groovy index 9550a7fec3dbd2..56f5d655255049 100644 --- a/regression-test/suites/query_p0/show/test_show_create_materialized_view.groovy +++ b/regression-test/suites/query_p0/show/test_show_create_materialized_view.groovy @@ -30,12 +30,11 @@ suite("test_show_create_materialized_view", "query,arrow_flight_sql") { DISTRIBUTED BY HASH(id) BUCKETS 5 PROPERTIES ("replication_num" = "1"); """ - - createMV("""CREATE MATERIALIZED VIEW ${mvName} AS + + create_sync_mv(context.dbName, tableName, mvName, """ SELECT id, name, SUM(value) AS total_value FROM ${tableName} - GROUP BY id, name; - """) + GROUP BY id, name""") checkNereidsExecute("""SHOW CREATE MATERIALIZED VIEW ${mvName} ON ${tableName};""") qt_cmd("""SHOW CREATE MATERIALIZED VIEW ${mvName} ON ${tableName};""") diff --git a/regression-test/suites/query_p0/system/test_partitions_schema.groovy b/regression-test/suites/query_p0/system/test_partitions_schema.groovy index 98b12c3705c777..84bbeafa7fe349 100644 --- a/regression-test/suites/query_p0/system/test_partitions_schema.groovy +++ b/regression-test/suites/query_p0/system/test_partitions_schema.groovy @@ -176,7 +176,7 @@ suite("test_partitions_schema") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } sql "GRANT SELECT_PRIV ON information_schema.partitions TO ${user}" diff --git a/regression-test/suites/query_p0/system/test_table_options.groovy b/regression-test/suites/query_p0/system/test_table_options.groovy index fef118a82b280b..fd9cffcedfcad8 100644 --- a/regression-test/suites/query_p0/system/test_table_options.groovy +++ b/regression-test/suites/query_p0/system/test_table_options.groovy @@ -191,7 +191,7 @@ suite("test_table_options") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } sql "GRANT SELECT_PRIV ON information_schema.table_properties TO ${user}" diff --git a/regression-test/suites/query_p0/system/test_table_properties.groovy b/regression-test/suites/query_p0/system/test_table_properties.groovy index 1861ae4d6280d6..7465497ae76d08 100644 --- a/regression-test/suites/query_p0/system/test_table_properties.groovy +++ b/regression-test/suites/query_p0/system/test_table_properties.groovy @@ -98,7 +98,7 @@ suite("test_table_properties") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } sql "GRANT SELECT_PRIV ON information_schema.table_properties TO ${user}" diff --git a/regression-test/suites/query_p0/test_row_policy.groovy b/regression-test/suites/query_p0/test_row_policy.groovy index 4af498e55e3b46..c416537df15eac 100644 --- a/regression-test/suites/query_p0/test_row_policy.groovy +++ b/regression-test/suites/query_p0/test_row_policy.groovy @@ -34,7 +34,7 @@ suite("test_row_policy") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } connect(user, '123456', url) { diff --git a/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy b/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy index 713c470436e7fb..a6ad20ec623048 100644 --- a/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy @@ -70,7 +70,7 @@ suite ("test_dup_mv_schema_change") { """ //add materialized view - createMV("create materialized view mv1 as select date, user_id, city, age from ${tableName};") + create_sync_mv(context.dbName, tableName, "mv1", """select date, user_id, city, age from ${tableName}""") // alter and test light schema change if (!isCloudMode()) { @@ -78,7 +78,7 @@ suite ("test_dup_mv_schema_change") { } //add materialized view - createMV("create materialized view mv2 as select date, user_id, city, age, cost from ${tableName};") + create_sync_mv(context.dbName, tableName, "mv2", """select date, user_id, city, age, cost from ${tableName}""") sql """ INSERT INTO ${tableName} VALUES (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 21) diff --git a/regression-test/suites/schema_change_p0/test_schema_change_with_mow_txn_conflict.groovy b/regression-test/suites/schema_change_p0/test_schema_change_with_mow_txn_conflict.groovy index 448ae3bc0fbea1..f2130956ad6b4e 100644 --- a/regression-test/suites/schema_change_p0/test_schema_change_with_mow_txn_conflict.groovy +++ b/regression-test/suites/schema_change_p0/test_schema_change_with_mow_txn_conflict.groovy @@ -18,7 +18,7 @@ import java.util.concurrent.TimeUnit import org.awaitility.Awaitility -suite("test_schema_change_with_mow_txn_conflict", "p0") { +suite("test_schema_change_with_mow_txn_conflict", "nonConcurrent") { def customFeConfig = [ schema_change_max_retry_time: 10 ] diff --git a/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy b/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy index eba6036c30a628..1d8fdd4d1e7192 100644 --- a/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy @@ -21,20 +21,7 @@ import org.awaitility.Awaitility suite ("test_uniq_mv_schema_change") { def tableName = "schema_change_uniq_mv_regression_test" - def getMVJobState = { tbName -> - def jobStateResult = sql """ SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName='${tbName}' ORDER BY CreateTime DESC LIMIT 1 """ - return jobStateResult[0][8] - } - def waitForJob = (tbName, timeout) -> { - Awaitility.await().atMost(timeout, TimeUnit.SECONDS).with().pollDelay(100, TimeUnit.MILLISECONDS).await().until(() -> { - String result = getMVJobState(tbName) - if (result == "FINISHED") { - return true; - } - return false; - }); - // when timeout awaitlity will raise a exception. - } + try { String backend_id; @@ -78,8 +65,7 @@ suite ("test_uniq_mv_schema_change") { //add materialized view def mvName = "mv1" - sql "create materialized view ${mvName} as select user_id, date, city, age, sex from ${tableName};" - waitForJob(tableName, 3000) + create_sync_mv(context.dbName, tableName, mvName, """select user_id, date, city, age, sex from ${tableName}""") // alter and test light schema change if (!isCloudMode()) { @@ -88,8 +74,7 @@ suite ("test_uniq_mv_schema_change") { //add materialized view def mvName2 = "mv2" - sql "create materialized view ${mvName2} as select user_id, date, city, age, sex, cost from ${tableName};" - waitForJob(tableName, 3000) + create_sync_mv(context.dbName, tableName, mvName2, """select user_id, date, city, age, sex, cost from ${tableName};""") sql """ INSERT INTO ${tableName} VALUES (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 21) diff --git a/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy b/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy index 56a0d7ddbf1f61..ae836fb0c65297 100644 --- a/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy +++ b/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy @@ -348,7 +348,7 @@ suite("test_crud_wlg") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO test_wlg_user"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO test_wlg_user"""; } connect('test_wlg_user', '12345', context.config.jdbcUrl) { @@ -739,7 +739,7 @@ suite("test_crud_wlg") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO test_wg_priv_user2"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO test_wg_priv_user2"""; } connect('test_wg_priv_user2', '', context.config.jdbcUrl) { qt_select_wgp_11 "select GRANTEE,WORKLOAD_GROUP_NAME,PRIVILEGE_TYPE,IS_GRANTABLE from information_schema.workload_group_privileges where grantee like '%test_wg_priv%' order by GRANTEE,WORKLOAD_GROUP_NAME,PRIVILEGE_TYPE,IS_GRANTABLE; " diff --git a/regression-test/suites/workload_manager_p0/test_resource_tag.groovy b/regression-test/suites/workload_manager_p0/test_resource_tag.groovy index fa7ba680143248..d643155c0eaeb2 100644 --- a/regression-test/suites/workload_manager_p0/test_resource_tag.groovy +++ b/regression-test/suites/workload_manager_p0/test_resource_tag.groovy @@ -25,7 +25,7 @@ suite("test_resource_tag") { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO test_rg"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO test_rg"""; } // test query