From e7f5cb5dea775d54eb12eea3b991e63ed6aac708 Mon Sep 17 00:00:00 2001 From: xzhangxian1008 Date: Tue, 22 Aug 2023 14:03:34 +0800 Subject: [PATCH] Support range frame type (#7629) ref pingcap/tiflash#7376 --- dbms/src/Core/Field.h | 10 +- .../DataStreams/WindowBlockInputStream.cpp | 677 ++++++++++++- dbms/src/DataStreams/WindowBlockInputStream.h | 79 +- dbms/src/DataTypes/getMostSubtype.cpp | 26 +- .../Coprocessor/DAGExpressionAnalyzer.cpp | 225 +++- dbms/src/Flash/Mpp/MPPTaskManager.h | 7 +- dbms/src/Functions/FunctionBinaryArithmetic.h | 2 +- dbms/src/Functions/FunctionsTiDBConversion.h | 12 +- dbms/src/Interpreters/WindowDescription.cpp | 6 +- dbms/src/Interpreters/WindowDescription.h | 20 + dbms/src/TestUtils/WindowTestUtils.h | 13 +- .../tests/gtest_first_value.cpp | 379 ++++++- .../tests/gtest_last_value.cpp | 323 +++++- .../WindowFunctions/tests/gtest_lead_lag.cpp | 5 +- .../mpp/window_range_frame.test | 957 ++++++++++++++++++ 15 files changed, 2628 insertions(+), 113 deletions(-) create mode 100644 tests/fullstack-test/mpp/window_range_frame.test diff --git a/dbms/src/Core/Field.h b/dbms/src/Core/Field.h index 638aeecbec3..420e113a3f2 100644 --- a/dbms/src/Core/Field.h +++ b/dbms/src/Core/Field.h @@ -377,7 +377,7 @@ class Field T & get() { using TWithoutRef = std::remove_reference_t; - TWithoutRef * MAY_ALIAS ptr = reinterpret_cast(&storage); + auto * MAY_ALIAS ptr = reinterpret_cast(&storage); return *ptr; }; @@ -385,7 +385,7 @@ class Field const T & get() const { using TWithoutRef = std::remove_reference_t; - const TWithoutRef * MAY_ALIAS ptr = reinterpret_cast(&storage); + const auto * MAY_ALIAS ptr = reinterpret_cast(&storage); return *ptr; }; @@ -582,7 +582,7 @@ class Field void createConcrete(T && x) { using JustT = std::decay_t; - JustT * MAY_ALIAS ptr = reinterpret_cast(&storage); + auto * MAY_ALIAS ptr = reinterpret_cast(&storage); new (ptr) JustT(std::forward(x)); which = TypeToEnum::value; } @@ -592,7 +592,7 @@ class Field void assignConcrete(T && x) { using JustT = std::decay_t; - JustT * MAY_ALIAS ptr = reinterpret_cast(&storage); + auto * MAY_ALIAS ptr = reinterpret_cast(&storage); *ptr = std::forward(x); } @@ -669,7 +669,7 @@ class Field void create(const char * data, size_t size) { - String * MAY_ALIAS ptr = reinterpret_cast(&storage); + auto * MAY_ALIAS ptr = reinterpret_cast(&storage); new (ptr) String(data, size); which = Types::String; } diff --git a/dbms/src/DataStreams/WindowBlockInputStream.cpp b/dbms/src/DataStreams/WindowBlockInputStream.cpp index acc03c65223..e4d8764aee4 100644 --- a/dbms/src/DataStreams/WindowBlockInputStream.cpp +++ b/dbms/src/DataStreams/WindowBlockInputStream.cpp @@ -12,9 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include +#include +#include +#include #include #include @@ -28,6 +40,206 @@ extern const int BAD_ARGUMENTS; extern const int NOT_IMPLEMENTED; } // namespace ErrorCodes +namespace +{ +template +consteval bool checkIfSimpleNumericType() +{ + return std::is_integral_v || std::is_floating_point_v; +} + +template +consteval bool checkIfDecimalFieldType() +{ + return std::is_same_v< + T, + DecimalField< + Decimal32>> || std::is_same_v> || std::is_same_v> || std::is_same_v>; +} + +template +bool lessEqual(LeftType left, RightType right) +{ + if constexpr (checkIfDecimalFieldType() && checkIfDecimalFieldType()) + { + return left <= right; + } + else if constexpr (checkIfDecimalFieldType()) + { + return DecimalComparison::compare( + left.getValue(), + right, + left.getScale(), + 0); + } + else if constexpr (checkIfDecimalFieldType()) + { + return DecimalComparison::compare( + left, + right.getValue(), + 0, + right.getScale()); + } + else + { + return left <= right; + } +} + +template +bool greaterEqual(LeftType left, RightType right) +{ + if constexpr (checkIfDecimalFieldType() && checkIfDecimalFieldType()) + { + return left >= right; + } + else if constexpr (checkIfDecimalFieldType()) + { + return DecimalComparison::compare( + left.getValue(), + right, + left.getScale(), + 0); + } + else if constexpr (checkIfDecimalFieldType()) + { + return DecimalComparison::compare( + left, + right.getValue(), + 0, + right.getScale()); + } + else + { + return left >= right; + } +} + +// When T is Decimal, we should convert it to DecimalField type +// as we need scale value when executing the comparison operation. +template +struct ActualCmpDataType +{ + using Type = std::conditional_t(), T, DecimalField>; +}; + +template +typename ActualCmpDataType::Type getValue(const ColumnPtr & col_ptr, size_t idx) +{ + return (*col_ptr)[idx].get::Type>(); +} + +template +bool isInRangeCommonImpl(T current_row_aux_value, U cursor_value) +{ + if constexpr (is_begin) + { + if constexpr (is_desc) + return lessEqual(cursor_value, current_row_aux_value); + else + return greaterEqual(cursor_value, current_row_aux_value); + } + else + { + if constexpr (!is_desc) + return lessEqual(cursor_value, current_row_aux_value); + else + return greaterEqual(cursor_value, current_row_aux_value); + } +} + +template +bool isInRangeIntImpl(T current_row_aux_value, U cursor_value) +{ + return isInRangeCommonImpl(current_row_aux_value, cursor_value); +} + +template +bool isInRangeDecimalImpl(AuxColType current_row_aux_value, OrderByColType cursor_value) +{ + return isInRangeCommonImpl( + current_row_aux_value, + cursor_value); +} + +template +bool isInRangeFloatImpl(AuxColType current_row_aux_value, OrderByColType cursor_value) +{ + Float64 current_row_aux_value_float64; + Float64 cursor_value_float64; + + if constexpr (checkIfDecimalFieldType()) + current_row_aux_value_float64 + = current_row_aux_value.getValue().template toFloat(current_row_aux_value.getScale()); + else + current_row_aux_value_float64 = static_cast(current_row_aux_value); + + if constexpr (checkIfDecimalFieldType()) + cursor_value_float64 = cursor_value.getValue().template toFloat(cursor_value.getScale()); + else + cursor_value_float64 = static_cast(cursor_value); + + return isInRangeCommonImpl( + current_row_aux_value_float64, + cursor_value_float64); +} + +template +bool isInRange(AuxColType current_row_aux_value, OrderByColType cursor_value) +{ + if constexpr (CmpDataType == tipb::RangeCmpDataType::Int) + { + // Two operand must be integer + if constexpr (std::is_integral_v && std::is_integral_v) + { + if constexpr (std::is_unsigned_v && std::is_unsigned_v) + return isInRangeIntImpl( + current_row_aux_value, + cursor_value); + return isInRangeIntImpl(current_row_aux_value, cursor_value); + } + else + throw Exception("Unexpected Data Type!"); + } + else if constexpr (CmpDataType == tipb::RangeCmpDataType::Float) + { + return isInRangeFloatImpl( + current_row_aux_value, + cursor_value); + } + else + { + if constexpr (std::is_floating_point_v || std::is_floating_point_v) + throw Exception("Occurrence of float type at here is unexpected!"); + else if constexpr (!checkIfDecimalFieldType() && !checkIfDecimalFieldType()) + throw Exception("At least one Decimal type is required"); + else + return isInRangeDecimalImpl( + current_row_aux_value, + cursor_value); + } +} + +template +RowNumber getBoundary(const WindowTransformAction & action) +{ + if constexpr (is_begin) + { + if (action.window_description.frame.begin_preceding) + return action.current_row; + else + return action.partition_end; + } + else + { + if (action.window_description.frame.end_preceding) + return action.current_row; + else + return action.partition_end; + } +} +} // namespace + WindowTransformAction::WindowTransformAction( const Block & input_header, const WindowDescription & window_description_, @@ -284,7 +496,7 @@ Int64 WindowTransformAction::getPartitionEndRow(size_t block_rows) // may haven't appeared and we can't find frame start in this case. // Returning false in the tuple's second parameter means the failure // of finding frame start. -std::tuple WindowTransformAction::stepToFrameStart( +std::tuple WindowTransformAction::stepToStartForRowsFrame( const RowNumber & current_row, const WindowFrame & frame) { @@ -295,7 +507,7 @@ std::tuple WindowTransformAction::stepToFrameStart( return stepInFollowing(current_row, step_num); } -std::tuple WindowTransformAction::stepToFrameEnd( +std::tuple WindowTransformAction::stepToEndForRowsFrame( const RowNumber & current_row, const WindowFrame & frame) { @@ -358,6 +570,9 @@ RowNumber WindowTransformAction::stepInPreceding(const RowNumber & moved_row, si std::tuple WindowTransformAction::stepInFollowing(const RowNumber & moved_row, size_t step_num) { if (!partition_ended) + // If we find the frame end and the partition_ended is false. + // The prev_frame_start may be equal to partition_end which + // will cause the assert fail in advancePartitionEnd function. return std::make_tuple(RowNumber(), false); auto dist = distance(partition_end, moved_row); @@ -400,6 +615,428 @@ std::tuple WindowTransformAction::stepInFollowing(const RowNumb return std::make_tuple(result_row, true); } +void WindowTransformAction::stepToFrameStart() +{ + RowNumber frame_start_tmp; + switch (window_description.frame.type) + { + case WindowFrame::FrameType::Rows: + { + std::tie(frame_start_tmp, frame_started) = stepToStartForRowsFrame(current_row, window_description.frame); + if (frame_started) + frame_start = frame_start_tmp; + break; + } + case WindowFrame::FrameType::Ranges: + { + std::tie(frame_start_tmp, frame_started) = stepToStartForRangeFrame(); + if (frame_started) + frame_start = frame_start_tmp; + break; + } + case WindowFrame::FrameType::Groups: + default: + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "window function only support frame type row and range."); + } +} + +void WindowTransformAction::stepToFrameEnd() +{ + switch (window_description.frame.type) + { + case WindowFrame::FrameType::Rows: + std::tie(frame_end, frame_ended) = stepToEndForRowsFrame(current_row, window_description.frame); + break; + case WindowFrame::FrameType::Ranges: + std::tie(frame_end, frame_ended) = stepToEndForRangeFrame(); + break; + case WindowFrame::FrameType::Groups: + default: + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "window function only support frame type row and range."); + } +} + +std::tuple WindowTransformAction::stepToStartForRangeFrame() +{ + if (!window_description.frame.begin_preceding && !partition_ended) + // If we find the frame end and the partition_ended is false. + // The prev_frame_start may be equal to partition_end which + // will cause the assert fail in advancePartitionEnd function. + return std::make_tuple(RowNumber(), false); + + if (window_description.is_desc) + return std::make_tuple(stepToStartForRangeFrameOrderCase(), true); + else + return std::make_tuple(stepToStartForRangeFrameOrderCase(), true); +} + +std::tuple WindowTransformAction::stepToEndForRangeFrame() +{ + if (!window_description.frame.end_preceding && !partition_ended) + // If we find the frame end and the partition_ended is false. + // Some previous blocks may be dropped, this is an unexpected behaviour. + // So, we shouldn't do anything before the partition_ended is true. + return std::make_tuple(RowNumber(), false); + + if (window_description.is_desc) + return stepToEndForRangeFrameOrderCase(); + else + return stepToEndForRangeFrameOrderCase(); +} + +template +RowNumber WindowTransformAction::stepToStartForRangeFrameOrderCase() +{ + switch (window_description.begin_aux_col_type) + { + case TypeIndex::UInt8: + return stepToStartForRangeFrameImpl(); + case TypeIndex::UInt16: + return stepToStartForRangeFrameImpl(); + case TypeIndex::UInt32: + return stepToStartForRangeFrameImpl(); + case TypeIndex::UInt64: + case TypeIndex::MyDate: + case TypeIndex::MyDateTime: + return stepToStartForRangeFrameImpl(); + case TypeIndex::Int8: + return stepToStartForRangeFrameImpl(); + case TypeIndex::Int16: + return stepToStartForRangeFrameImpl(); + case TypeIndex::Int32: + return stepToStartForRangeFrameImpl(); + case TypeIndex::Int64: + case TypeIndex::MyTime: + return stepToStartForRangeFrameImpl(); + case TypeIndex::Float32: + return stepToStartForRangeFrameImpl(); + case TypeIndex::Float64: + return stepToStartForRangeFrameImpl(); + case TypeIndex::Decimal32: + return stepToStartForRangeFrameImpl(); + case TypeIndex::Decimal64: + return stepToStartForRangeFrameImpl(); + case TypeIndex::Decimal128: + return stepToStartForRangeFrameImpl(); + case TypeIndex::Decimal256: + return stepToStartForRangeFrameImpl(); + default: + throw Exception("Unexpected column type!"); + } +} + +template +std::tuple WindowTransformAction::stepToEndForRangeFrameOrderCase() +{ + switch (window_description.end_aux_col_type) + { + case TypeIndex::UInt8: + return std::make_tuple(stepToEndForRangeFrameImpl(), true); + case TypeIndex::UInt16: + return std::make_tuple(stepToEndForRangeFrameImpl(), true); + case TypeIndex::UInt32: + return std::make_tuple(stepToEndForRangeFrameImpl(), true); + case TypeIndex::UInt64: + case TypeIndex::MyDate: + case TypeIndex::MyDateTime: + return std::make_tuple(stepToEndForRangeFrameImpl(), true); + case TypeIndex::Int8: + return std::make_tuple(stepToEndForRangeFrameImpl(), true); + case TypeIndex::Int16: + return std::make_tuple(stepToEndForRangeFrameImpl(), true); + case TypeIndex::Int32: + return std::make_tuple(stepToEndForRangeFrameImpl(), true); + case TypeIndex::Int64: + case TypeIndex::MyTime: + return std::make_tuple(stepToEndForRangeFrameImpl(), true); + case TypeIndex::Float32: + return std::make_tuple(stepToEndForRangeFrameImpl(), true); + case TypeIndex::Float64: + return std::make_tuple(stepToEndForRangeFrameImpl(), true); + case TypeIndex::Decimal32: + return std::make_tuple(stepToEndForRangeFrameImpl(), true); + case TypeIndex::Decimal64: + return std::make_tuple(stepToEndForRangeFrameImpl(), true); + case TypeIndex::Decimal128: + return std::make_tuple(stepToEndForRangeFrameImpl(), true); + case TypeIndex::Decimal256: + return std::make_tuple(stepToEndForRangeFrameImpl(), true); + default: + throw Exception("Unexpected column type!"); + } +} + +template +RowNumber WindowTransformAction::stepToStartForRangeFrameImpl() +{ + return stepForRangeFrameImpl(); +} + +template +RowNumber WindowTransformAction::stepToEndForRangeFrameImpl() +{ + return stepForRangeFrameImpl(); +} + +template +RowNumber WindowTransformAction::stepForRangeFrameImpl() +{ + bool is_col_nullable; + if constexpr (is_begin) + is_col_nullable = window_description.is_begin_aux_col_nullable; + else + is_col_nullable = window_description.is_end_aux_col_nullable; + + if (is_col_nullable) + { + ColumnPtr order_by_column = inputAt(current_row)[order_column_indices[0]]; + if (order_by_column->isNullAt(current_row.row)) + return findRangeFrameIfNull(current_row); + } + + RowNumber cursor; + if constexpr (is_begin) + cursor = prev_frame_start; + else + cursor = prev_frame_end; + + size_t cur_row_aux_col_idx; + if constexpr (is_begin) + cur_row_aux_col_idx = window_description.frame.begin_range_auxiliary_column_index; + else + cur_row_aux_col_idx = window_description.frame.end_range_auxiliary_column_index; + + ColumnPtr cur_row_aux_column = inputAt(current_row)[cur_row_aux_col_idx]; + typename ActualCmpDataType::Type current_row_aux_value = getValue(cur_row_aux_column, current_row.row); + return moveCursorAndFindRangeFrame::Type, is_begin, is_desc>( + cursor, + current_row_aux_value); +} + +template +RowNumber WindowTransformAction::findRangeFrameIfNull(RowNumber cursor) +{ + if (!is_range_null_frame_initialized) + { + // We always see the first cursor as frame start + range_null_frame_start = cursor; + + while (cursor < partition_end) + { + const ColumnPtr & cursor_column = inputAt(cursor)[order_column_indices[0]]; + if (!cursor_column->isNullAt(cursor.row)) + break; + advanceRowNumber(cursor); + } + + range_null_frame_end = cursor; + is_range_null_frame_initialized = true; + } + + if constexpr (is_begin) + return range_null_frame_start; + else + return range_null_frame_end; +} + +template +RowNumber WindowTransformAction::moveCursorAndFindRangeFrame(RowNumber cursor, AuxColType current_row_aux_value) +{ + switch (window_description.order_by_col_type) + { + case TypeIndex::UInt8: + return moveCursorAndFindRangeFrame(cursor, current_row_aux_value); + case TypeIndex::UInt16: + return moveCursorAndFindRangeFrame(cursor, current_row_aux_value); + case TypeIndex::UInt32: + return moveCursorAndFindRangeFrame(cursor, current_row_aux_value); + case TypeIndex::UInt64: + case TypeIndex::MyDate: + case TypeIndex::MyDateTime: + return moveCursorAndFindRangeFrame(cursor, current_row_aux_value); + case TypeIndex::Int8: + return moveCursorAndFindRangeFrame(cursor, current_row_aux_value); + case TypeIndex::Int16: + return moveCursorAndFindRangeFrame(cursor, current_row_aux_value); + case TypeIndex::Int32: + return moveCursorAndFindRangeFrame(cursor, current_row_aux_value); + case TypeIndex::Int64: + case TypeIndex::MyTime: + return moveCursorAndFindRangeFrame(cursor, current_row_aux_value); + case TypeIndex::Float32: + return moveCursorAndFindRangeFrame(cursor, current_row_aux_value); + case TypeIndex::Float64: + return moveCursorAndFindRangeFrame(cursor, current_row_aux_value); + case TypeIndex::Decimal32: + return moveCursorAndFindRangeFrame(cursor, current_row_aux_value); + case TypeIndex::Decimal64: + return moveCursorAndFindRangeFrame(cursor, current_row_aux_value); + case TypeIndex::Decimal128: + return moveCursorAndFindRangeFrame(cursor, current_row_aux_value); + case TypeIndex::Decimal256: + return moveCursorAndFindRangeFrame(cursor, current_row_aux_value); + default: + throw Exception("Unexpected column type!"); + } +} + +template +RowNumber WindowTransformAction::moveCursorAndFindRangeFrame(RowNumber cursor, AuxColType current_row_aux_value) +{ + tipb::RangeCmpDataType cmp_data_type; + if constexpr (is_begin) + cmp_data_type = window_description.frame.begin_cmp_data_type; + else + cmp_data_type = window_description.frame.end_cmp_data_type; + + if (window_description.is_order_by_col_nullable) + { + switch (cmp_data_type) + { + case tipb::RangeCmpDataType::Int: + return moveCursorAndFindRangeFrameImpl< + AuxColType, + OrderByColType, + tipb::RangeCmpDataType::Int, + is_begin, + is_desc, + true>(cursor, current_row_aux_value); + case tipb::RangeCmpDataType::Float: + return moveCursorAndFindRangeFrameImpl< + AuxColType, + OrderByColType, + tipb::RangeCmpDataType::Float, + is_begin, + is_desc, + true>(cursor, current_row_aux_value); + case tipb::RangeCmpDataType::Decimal: + return moveCursorAndFindRangeFrameImpl< + AuxColType, + OrderByColType, + tipb::RangeCmpDataType::Decimal, + is_begin, + is_desc, + true>(cursor, current_row_aux_value); + default: + throw Exception(fmt::format("Unexpected RangeCmpDataType: {}", magic_enum::enum_name(cmp_data_type))); + } + } + else + { + switch (cmp_data_type) + { + case tipb::RangeCmpDataType::Int: + return moveCursorAndFindRangeFrameImpl< + AuxColType, + OrderByColType, + tipb::RangeCmpDataType::Int, + is_begin, + is_desc, + false>(cursor, current_row_aux_value); + case tipb::RangeCmpDataType::Float: + return moveCursorAndFindRangeFrameImpl< + AuxColType, + OrderByColType, + tipb::RangeCmpDataType::Float, + is_begin, + is_desc, + false>(cursor, current_row_aux_value); + case tipb::RangeCmpDataType::Decimal: + return moveCursorAndFindRangeFrameImpl< + AuxColType, + OrderByColType, + tipb::RangeCmpDataType::Decimal, + is_begin, + is_desc, + false>(cursor, current_row_aux_value); + default: + throw Exception("Unexpected RangeCmpDataType!"); + } + } +} + +template < + typename AuxColType, + typename OrderByColType, + int CmpDataType, + bool is_begin, + bool is_desc, + bool is_order_by_col_nullable> +RowNumber WindowTransformAction::moveCursorAndFindRangeFrameImpl(RowNumber cursor, AuxColType current_row_aux_value) +{ + using ActualOrderByColType = typename ActualCmpDataType::Type; + + RowNumber boundary = getBoundary(*this); + while (cursor < boundary) + { + const ColumnPtr & cursor_column = inputAt(cursor)[order_column_indices[0]]; + if constexpr (is_order_by_col_nullable) + { + if (cursor_column->isNullAt(cursor.row)) + { + if constexpr (is_begin) + { + if (!is_desc) + advanceRowNumber(cursor); + else + return cursor; + continue; + } + else + { + if (window_description.frame.end_preceding) + { + advanceRowNumber(cursor); + continue; + } + else + return cursor; + } + } + } + + ActualOrderByColType cursor_value = getValue(cursor_column, cursor.row); + + if constexpr (is_begin) + { + if (window_description.frame.begin_preceding) + { + if (isInRange( + current_row_aux_value, + cursor_value)) + return cursor; + } + else + { + if (isInRange( + current_row_aux_value, + cursor_value)) + return cursor; + } + } + else + { + if (window_description.frame.end_preceding) + { + if (!isInRange( + current_row_aux_value, + cursor_value)) + return cursor; + } + else + { + if (!isInRange( + current_row_aux_value, + cursor_value)) + return cursor; + } + } + + advanceRowNumber(cursor); + } + return cursor; +} + UInt64 WindowTransformAction::distance(RowNumber left, RowNumber right) { if (left.block == right.block) @@ -443,19 +1080,7 @@ void WindowTransformAction::advanceFrameStart() break; } case WindowFrame::BoundaryType::Offset: - if (window_description.frame.type == WindowFrame::FrameType::Rows) - { - RowNumber frame_start_tmp; - std::tie(frame_start_tmp, frame_started) = stepToFrameStart(current_row, window_description.frame); - if (frame_started) - frame_start = frame_start_tmp; - } - else - throw Exception( - ErrorCodes::NOT_IMPLEMENTED, - fmt::format( - "Frame type {}'s Offset BoundaryType is not implemented", - magic_enum::enum_name(window_description.frame.type))); + stepToFrameStart(); break; default: throw Exception( @@ -537,14 +1162,6 @@ void WindowTransformAction::advanceFrameEndCurrentRow() void WindowTransformAction::advanceFrameEnd() { - // frame_end must be greater or equal than frame_start, so if the - // frame_start is already past the current frame_end, we can start - // from it to save us some work. - if (frame_end < frame_start) - { - frame_end = frame_start; - } - // No reason for this function to be called again after it succeeded. assert(!frame_ended); @@ -563,14 +1180,7 @@ void WindowTransformAction::advanceFrameEnd() } case WindowFrame::BoundaryType::Offset: { - if (window_description.frame.type == WindowFrame::FrameType::Rows) - std::tie(frame_end, frame_ended) = stepToFrameEnd(current_row, window_description.frame); - else - throw Exception( - ErrorCodes::NOT_IMPLEMENTED, - fmt::format( - "Frame type {}'s Offset BoundaryType is not implemented", - magic_enum::enum_name(window_description.frame.type))); + stepToFrameEnd(); break; } default: @@ -664,9 +1274,7 @@ void WindowTransformAction::appendBlock(Block & current_block) assert(current_block); if (current_block.rows() == 0) - { return; - } window_blocks.push_back({}); auto & window_block = window_blocks.back(); @@ -746,6 +1354,7 @@ void WindowTransformAction::tryCalculate() writeOutCurrentRow(); prev_frame_start = frame_start; + prev_frame_end = frame_end; // Move to the next row. The frame will have to be recalculated. // The peer group start is updated at the beginning of the loop, @@ -783,11 +1392,13 @@ void WindowTransformAction::tryCalculate() frame_start = partition_start; frame_end = partition_start; prev_frame_start = partition_start; + prev_frame_end = partition_end; assert(current_row == partition_start); current_row_number = 1; peer_group_last = partition_start; peer_group_start_row_number = 1; peer_group_number = 1; + is_range_null_frame_initialized = false; } } diff --git a/dbms/src/DataStreams/WindowBlockInputStream.h b/dbms/src/DataStreams/WindowBlockInputStream.h index cefec4ab36d..e46424e32ae 100644 --- a/dbms/src/DataStreams/WindowBlockInputStream.h +++ b/dbms/src/DataStreams/WindowBlockInputStream.h @@ -14,6 +14,7 @@ #pragma once +#include #include #include #include @@ -29,18 +30,6 @@ namespace DB /* Implementation details.*/ struct WindowTransformAction { -private: - // Used for calculating the frame start - std::tuple stepToFrameStart(const RowNumber & current_row, const WindowFrame & frame); - // Used for calculating the frame end - std::tuple stepToFrameEnd(const RowNumber & current_row, const WindowFrame & frame); - - RowNumber stepInPreceding(const RowNumber & moved_row, size_t step_num); - std::tuple stepInFollowing(const RowNumber & moved_row, size_t step_num); - - // distance is left - right. - UInt64 distance(RowNumber left, RowNumber right); - public: WindowTransformAction( const Block & input_header, @@ -120,6 +109,63 @@ struct WindowTransformAction void appendInfo(FmtBuffer & buffer) const; +private: + // This is the function for Offset type boundary + void stepToFrameStart(); + // This is the function for Offset type boundary + void stepToFrameEnd(); + + // Used for calculating the frame start for rows frame type + std::tuple stepToStartForRowsFrame(const RowNumber & current_row, const WindowFrame & frame); + // Used for calculating the frame end for rows frame type + std::tuple stepToEndForRowsFrame(const RowNumber & current_row, const WindowFrame & frame); + + // Used for calculating the frame start for range frame type + std::tuple stepToStartForRangeFrame(); + // Used for calculating the frame end for range frame type + std::tuple stepToEndForRangeFrame(); + + template + RowNumber stepToStartForRangeFrameOrderCase(); + + template + std::tuple stepToEndForRangeFrameOrderCase(); + + template + RowNumber stepToStartForRangeFrameImpl(); + + template + RowNumber stepToEndForRangeFrameImpl(); + + template + RowNumber stepForRangeFrameImpl(); + + // We should use this function when the current order by column row is null. + template + RowNumber findRangeFrameIfNull(RowNumber cursor); + + template + RowNumber moveCursorAndFindRangeFrame(RowNumber cursor, AuxColType current_row_aux_value); + + template + RowNumber moveCursorAndFindRangeFrame(RowNumber cursor, AuxColType current_row_aux_value); + + template < + typename AuxColType, + typename OrderByColType, + int CmpDataType, + bool is_begin, + bool is_desc, + bool is_order_by_col_nullable> + RowNumber moveCursorAndFindRangeFrameImpl(RowNumber cursor, AuxColType current_row_aux_value); + + RowNumber stepInPreceding(const RowNumber & moved_row, size_t step_num); + std::tuple stepInFollowing(const RowNumber & moved_row, size_t step_num); + + // distance is left - right. + UInt64 distance(RowNumber left, RowNumber right); + +public: LoggerPtr log; bool input_is_finished = false; @@ -130,7 +176,7 @@ struct WindowTransformAction // Indices of the PARTITION BY columns in block. std::vector partition_column_indices; - // Indices of the ORDER BY columns in block; + // Indices of the ORDER BY columns in block. std::vector order_column_indices; // Per-window-function scratch spaces. @@ -186,11 +232,18 @@ struct WindowTransformAction bool frame_ended = false; bool frame_started = false; + RowNumber range_null_frame_start; + RowNumber range_null_frame_end; + bool is_range_null_frame_initialized = false; + // The previous frame boundaries that correspond to the current state of the // aggregate function. We use them to determine how to update the aggregation // state after we find the new frame. RowNumber prev_frame_start; + // Auxiliary variable for range frame type when calculating frame_end + RowNumber prev_frame_end; + //TODO: used as template parameters bool only_have_row_number = false; }; diff --git a/dbms/src/DataTypes/getMostSubtype.cpp b/dbms/src/DataTypes/getMostSubtype.cpp index 89c98ea4572..4429a4f26f6 100644 --- a/dbms/src/DataTypes/getMostSubtype.cpp +++ b/dbms/src/DataTypes/getMostSubtype.cpp @@ -58,7 +58,7 @@ String getExceptionMessagePrefix(const DataTypes & types) DataTypePtr getMostSubtype(const DataTypes & types, bool throw_if_result_is_nothing, bool force_support_conversion) { - auto getNothingOrThrow = [throw_if_result_is_nothing, &types](const std::string & reason) { + auto get_nothing_or_throw = [throw_if_result_is_nothing, &types](const std::string & reason) { if (throw_if_result_is_nothing) throw Exception(getExceptionMessagePrefix(types) + reason, ErrorCodes::NO_COMMON_TYPE); return std::make_shared(); @@ -102,7 +102,7 @@ DataTypePtr getMostSubtype(const DataTypes & types, bool throw_if_result_is_noth { for (const auto & type : types) if (typeid_cast(type.get())) - return getNothingOrThrow(" because some of them are Nothing"); + return get_nothing_or_throw(" because some of them are Nothing"); } /// For Arrays @@ -115,7 +115,7 @@ DataTypePtr getMostSubtype(const DataTypes & types, bool throw_if_result_is_noth for (const auto & type : types) { - if (const auto type_array = typeid_cast(type.get())) + if (const auto * const type_array = typeid_cast(type.get())) { have_array = true; nested_types.emplace_back(type_array->getNestedType()); @@ -127,7 +127,7 @@ DataTypePtr getMostSubtype(const DataTypes & types, bool throw_if_result_is_noth if (have_array) { if (!all_arrays) - return getNothingOrThrow(" because some of them are Array and some of them are not"); + return get_nothing_or_throw(" because some of them are Array and some of them are not"); return std::make_shared(getMostSubtype(nested_types, false, force_support_conversion)); } @@ -143,7 +143,7 @@ DataTypePtr getMostSubtype(const DataTypes & types, bool throw_if_result_is_noth for (const auto & type : types) { - if (const auto type_tuple = typeid_cast(type.get())) + if (const auto * const type_tuple = typeid_cast(type.get())) { if (!have_tuple) { @@ -153,7 +153,7 @@ DataTypePtr getMostSubtype(const DataTypes & types, bool throw_if_result_is_noth nested_types[elem_idx].reserve(types.size()); } else if (tuple_size != type_tuple->getElements().size()) - return getNothingOrThrow(" because Tuples have different sizes"); + return get_nothing_or_throw(" because Tuples have different sizes"); have_tuple = true; @@ -167,7 +167,7 @@ DataTypePtr getMostSubtype(const DataTypes & types, bool throw_if_result_is_noth if (have_tuple) { if (!all_tuples) - return getNothingOrThrow(" because some of them are Tuple and some of them are not"); + return get_nothing_or_throw(" because some of them are Tuple and some of them are not"); DataTypes common_tuple_types(tuple_size); for (size_t elem_idx = 0; elem_idx < tuple_size; ++elem_idx) @@ -188,7 +188,7 @@ DataTypePtr getMostSubtype(const DataTypes & types, bool throw_if_result_is_noth for (const auto & type : types) { - if (const auto type_nullable = typeid_cast(type.get())) + if (const auto * const type_nullable = typeid_cast(type.get())) { have_nullable = true; nested_types.emplace_back(type_nullable->getNestedType()); @@ -229,7 +229,7 @@ DataTypePtr getMostSubtype(const DataTypes & types, bool throw_if_result_is_noth if (!fixed_string_type) fixed_string_type = type; else if (!type->equals(*fixed_string_type)) - return getNothingOrThrow(" because some of them are FixedStrings with different length"); + return get_nothing_or_throw(" because some of them are FixedStrings with different length"); } else if (type->isString()) have_string = true; @@ -240,7 +240,7 @@ DataTypePtr getMostSubtype(const DataTypes & types, bool throw_if_result_is_noth if (have_string) { if (!all_strings) - return getNothingOrThrow(" because some of them are String/FixedString and some of them are not"); + return get_nothing_or_throw(" because some of them are String/FixedString and some of them are not"); return fixed_string_type ? fixed_string_type : std::make_shared(); } @@ -262,7 +262,7 @@ DataTypePtr getMostSubtype(const DataTypes & types, bool throw_if_result_is_noth if (have_date_or_datetime) { if (!all_date_or_datetime) - return getNothingOrThrow(" because some of them are Date/DateTime and some of them are not"); + return get_nothing_or_throw(" because some of them are Date/DateTime and some of them are not"); return std::make_shared(); } @@ -310,7 +310,7 @@ DataTypePtr getMostSubtype(const DataTypes & types, bool throw_if_result_is_noth if (min_bits_of_signed_integer || min_bits_of_unsigned_integer || min_mantissa_bits_of_floating) { if (!all_numbers) - return getNothingOrThrow(" because some of them are numbers and some of them are not"); + return get_nothing_or_throw(" because some of them are numbers and some of them are not"); /// If the result must be floating. if (!min_bits_of_signed_integer && !min_bits_of_unsigned_integer) @@ -365,7 +365,7 @@ DataTypePtr getMostSubtype(const DataTypes & types, bool throw_if_result_is_noth } /// All other data types (UUID, AggregateFunction, Enum...) are compatible only if they are the same (checked in trivial cases). - return getNothingOrThrow(""); + return get_nothing_or_throw(""); } } // namespace DB diff --git a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.cpp b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.cpp index eb810e4bb8a..e52e681da53 100644 --- a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.cpp +++ b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.cpp @@ -15,9 +15,12 @@ #include #include #include +#include +#include #include #include #include +#include #include #include #include @@ -31,14 +34,19 @@ #include #include #include +#include +#include #include #include +#include #include #include #include #include #include #include +#include +#include namespace DB @@ -195,6 +203,188 @@ void appendWindowDescription( window_columns.emplace_back(func_string, result_type); source_columns.emplace_back(func_string, result_type); } + +void setAuxiliaryColumnInfoImpl( + const String & aux_col_name, + const Block & tmp_block, + Int32 & range_auxiliary_column_index, + TypeIndex & aux_col_type, + bool & is_order_by_col_nullable) +{ + if (!aux_col_name.empty()) + { + // Set auxiliary columns' indexes + size_t aux_col_idx = tmp_block.getPositionByName(aux_col_name); + range_auxiliary_column_index = aux_col_idx; + + // Set auxiliary columns' types + const auto & col_and_name = tmp_block.getByName(aux_col_name); + auto data_type = col_and_name.type; + if (data_type->isNullable()) + { + is_order_by_col_nullable = true; + const auto & nullable_data_type = static_cast(*data_type); + aux_col_type = nullable_data_type.getNestedType()->getTypeId(); + } + else + { + is_order_by_col_nullable = false; + aux_col_type = data_type->getTypeId(); + } + } +} + +// We need auxiliary columns' info when finding the start or end boundary of the frame +void setAuxiliaryColumnInfo( + ExpressionActionsPtr & actions, + WindowDescription & window_desc, + const String & begin_aux_col_name, + const String & end_aux_col_name, + const tipb::Window & window) +{ + // Execute this function only when the frame type is Range + if (window.frame().type() != tipb::WindowFrameType::Ranges) + return; + + if (begin_aux_col_name.empty() && end_aux_col_name.empty()) + return; + + const Block & tmp_block = actions->getSampleBlock(); + if (!begin_aux_col_name.empty()) + setAuxiliaryColumnInfoImpl( + begin_aux_col_name, + tmp_block, + window_desc.frame.begin_range_auxiliary_column_index, + window_desc.begin_aux_col_type, + window_desc.is_begin_aux_col_nullable); + if (!end_aux_col_name.empty()) + setAuxiliaryColumnInfoImpl( + end_aux_col_name, + tmp_block, + window_desc.frame.end_range_auxiliary_column_index, + window_desc.end_aux_col_type, + window_desc.is_end_aux_col_nullable); +} + +void setOrderByColumnTypeAndDirectionForRangeFrame( + WindowDescription & window_desc, + const ExpressionActionsPtr & actions, + const tipb::Window & window) +{ + // Execute this function only when the frame type is Range + if (window.frame().type() != tipb::WindowFrameType::Ranges) + return; + + RUNTIME_CHECK_MSG( + !window_desc.order_by.empty(), + "Order by column should not be empty when the frame type is range"); + RUNTIME_CHECK_MSG( + window_desc.order_by.size() == 1, + "Number of order by should not be larger than 1 in range frame"); + + const Block & sample_block = actions->getSampleBlock(); + const String & order_by_col_name = window_desc.order_by[0].column_name; + const ColumnWithTypeAndName & order_by_col_type_and_name = sample_block.getByName(order_by_col_name); + + if (order_by_col_type_and_name.type->isNullable()) + { + window_desc.is_order_by_col_nullable = true; + const auto & nullable_data_type = static_cast(*order_by_col_type_and_name.type); + window_desc.order_by_col_type = nullable_data_type.getNestedType()->getTypeId(); + } + else + { + window_desc.is_order_by_col_nullable = false; + window_desc.order_by_col_type = order_by_col_type_and_name.type->getTypeId(); + } + window_desc.is_desc = (window_desc.order_by[0].direction == -1); +} + +// Add a function generating a new auxiliary column that help the implementation of range frame type +std::pair addRangeFrameAuxiliaryFunctionAction( + DAGExpressionAnalyzer * analyzer, + ExpressionActionsPtr & actions, + const tipb::Window & window) +{ + // Execute this function only when the frame type is Range + if (window.frame().type() != tipb::WindowFrameType::Ranges) + return std::make_pair("", ""); + + RUNTIME_CHECK_MSG( + window.frame().start().has_frame_range() || window.frame().end().has_frame_range(), + "tipb::WindowFrameBound of start or end must be set when the frame type is range"); + + String begin_aux_col_name; + String end_aux_col_name; + if (window.frame().start().has_frame_range()) + begin_aux_col_name + = DAGExpressionAnalyzerHelper::buildFunction(analyzer, window.frame().start().frame_range(), actions); + + if (window.frame().end().has_frame_range()) + end_aux_col_name + = DAGExpressionAnalyzerHelper::buildFunction(analyzer, window.frame().end().frame_range(), actions); + + return std::make_pair(begin_aux_col_name, end_aux_col_name); +} + +WindowDescription createAndInitWindowDesc(DAGExpressionAnalyzer * const analyzer, const tipb::Window & window) +{ + WindowDescription window_description; + window_description.partition_by = analyzer->getWindowSortDescription(window.partition_by()); + window_description.order_by = analyzer->getWindowSortDescription(window.order_by()); + if (window.has_frame()) + { + window_description.setWindowFrame(window.frame()); + } + + return window_description; +} + +void buildActionsBeforeWindow( + DAGExpressionAnalyzer * analyzer, + WindowDescription & window_desc, + ExpressionActionsChain & chain, + const tipb::Window & window) +{ + auto actions = chain.getLastActions(); + + // Prepare auxiliary function for range frame type + auto aux_col_names = addRangeFrameAuxiliaryFunctionAction(analyzer, actions, window); + + analyzer->appendWindowColumns(window_desc, window, actions); + // set required output for window funcs's arguments. + for (const auto & window_function_description : window_desc.window_functions_descriptions) + { + for (const auto & argument_name : window_function_description.argument_names) + chain.getLastStep().required_output.push_back(argument_name); + } + + window_desc.before_window = actions; + if (!aux_col_names.first.empty()) + chain.getLastStep().required_output.push_back(aux_col_names.first); + if (!aux_col_names.second.empty()) + chain.getLastStep().required_output.push_back(aux_col_names.second); + + chain.finalize(); + chain.clear(); + setAuxiliaryColumnInfo(actions, window_desc, aux_col_names.first, aux_col_names.second, window); +} + +void buildActionsAfterWindow( + DAGExpressionAnalyzer * const analyze, + WindowDescription & window_desc, + ExpressionActionsChain & chain, + const tipb::Window & window, + size_t source_size) +{ + auto & after_window_step = analyze->initAndGetLastStep(chain); + analyze->appendCastAfterWindow(after_window_step.actions, window, source_size); + window_desc.after_window_columns = analyze->getCurrentInputColumns(); + analyze->appendSourceColumnsToRequireOutput(after_window_step); + window_desc.after_window = chain.getLastActions(); + chain.finalize(); + chain.clear(); +} } // namespace ExpressionActionsChain::Step & DAGExpressionAnalyzer::initAndGetLastStep(ExpressionActionsChain & chain) const @@ -685,36 +875,13 @@ WindowDescription DAGExpressionAnalyzer::buildWindowDescription(const tipb::Wind ExpressionActionsChain chain; ExpressionActionsChain::Step & step = initAndGetLastStep(chain); appendSourceColumnsToRequireOutput(step); - size_t source_size = getCurrentInputColumns().size(); - - WindowDescription window_description; - window_description.partition_by = getWindowSortDescription(window.partition_by()); - window_description.order_by = getWindowSortDescription(window.order_by()); - if (window.has_frame()) - { - window_description.setWindowFrame(window.frame()); - } - - appendWindowColumns(window_description, window, step.actions); - // set required output for window funcs's arguments. - for (const auto & window_function_description : window_description.window_functions_descriptions) - { - for (const auto & argument_name : window_function_description.argument_names) - step.required_output.push_back(argument_name); - } - - window_description.before_window = chain.getLastActions(); - chain.finalize(); - chain.clear(); + size_t source_size = getCurrentInputColumns().size(); - auto & after_window_step = initAndGetLastStep(chain); - appendCastAfterWindow(after_window_step.actions, window, source_size); - window_description.after_window_columns = getCurrentInputColumns(); - appendSourceColumnsToRequireOutput(after_window_step); - window_description.after_window = chain.getLastActions(); - chain.finalize(); - chain.clear(); + WindowDescription window_description = createAndInitWindowDesc(this, window); + setOrderByColumnTypeAndDirectionForRangeFrame(window_description, step.actions, window); + buildActionsBeforeWindow(this, window_description, chain, window); + buildActionsAfterWindow(this, window_description, chain, window, source_size); return window_description; } @@ -1499,8 +1666,8 @@ void DAGExpressionAnalyzer::initChain(ExpressionActionsChain & chain) const if (chain.steps.empty()) { const auto & columns = getCurrentInputColumns(); - NamesAndTypesList column_list; std::unordered_set column_name_set; + NamesAndTypesList column_list; for (const auto & col : columns) { if (column_name_set.find(col.name) == column_name_set.end()) diff --git a/dbms/src/Flash/Mpp/MPPTaskManager.h b/dbms/src/Flash/Mpp/MPPTaskManager.h index c43a8a7a14f..f39d92ccc37 100644 --- a/dbms/src/Flash/Mpp/MPPTaskManager.h +++ b/dbms/src/Flash/Mpp/MPPTaskManager.h @@ -150,7 +150,10 @@ struct MPPTaskMonitor auto iter = monitored_tasks.find(task_unique_id); if (iter != monitored_tasks.end()) { - LOG_WARNING(log, "task {} is repeatedly added to be monitored which is not an expected behavior!"); + LOG_WARNING( + log, + "task {} is repeatedly added to be monitored which is not an expected behavior!", + task_unique_id); return; } @@ -163,7 +166,7 @@ struct MPPTaskMonitor auto iter = monitored_tasks.find(task_unique_id); if (iter == monitored_tasks.end()) { - LOG_WARNING(log, "Unexpected behavior! task {} is not found in monitored_task."); + LOG_WARNING(log, "Unexpected behavior! task {} is not found in monitored_task.", task_unique_id); return; } diff --git a/dbms/src/Functions/FunctionBinaryArithmetic.h b/dbms/src/Functions/FunctionBinaryArithmetic.h index 4cc378ea219..16e262edda6 100644 --- a/dbms/src/Functions/FunctionBinaryArithmetic.h +++ b/dbms/src/Functions/FunctionBinaryArithmetic.h @@ -926,7 +926,7 @@ class FunctionBinaryArithmetic : public IFunction int interval_arg = 1; /// do not check null type because only divide op may use non-default-impl for nulls - const DataTypeInterval * interval_data_type = checkAndGetDataType(type1.get()); + const auto * interval_data_type = checkAndGetDataType(type1.get()); if (!interval_data_type) { interval_arg = 0; diff --git a/dbms/src/Functions/FunctionsTiDBConversion.h b/dbms/src/Functions/FunctionsTiDBConversion.h index 704a79eb92d..ccad0d60a12 100644 --- a/dbms/src/Functions/FunctionsTiDBConversion.h +++ b/dbms/src/Functions/FunctionsTiDBConversion.h @@ -134,7 +134,7 @@ struct TiDBConvertToString { /// cast string as string const IColumn * col_from = block.getByPosition(arguments[0]).column.get(); - const ColumnString * col_from_string = checkAndGetColumn(col_from); + const auto * col_from_string = checkAndGetColumn(col_from); const ColumnString::Chars_t * data_from = &col_from_string->getChars(); const IColumn::Offsets * offsets_from = &col_from_string->getOffsets(); @@ -537,7 +537,7 @@ struct TiDBConvertToInteger { /// cast string as int const IColumn * col_from = block.getByPosition(arguments[0]).column.get(); - const ColumnString * col_from_string = checkAndGetColumn(col_from); + const auto * col_from_string = checkAndGetColumn(col_from); const ColumnString::Chars_t * chars = &col_from_string->getChars(); const IColumn::Offsets * offsets = &col_from_string->getOffsets(); size_t current_offset = 0; @@ -640,7 +640,7 @@ struct TiDBConvertToFloat Float64 max_f, const Context & context) { - Float64 float_value = static_cast(value); + auto float_value = static_cast(value); return produceTargetFloat64(float_value, need_truncate, shift, max_f, context); } @@ -804,7 +804,7 @@ struct TiDBConvertToFloat { /// cast string as real const IColumn * col_from = block.getByPosition(arguments[0]).column.get(); - const ColumnString * col_from_string = checkAndGetColumn(col_from); + const auto * col_from_string = checkAndGetColumn(col_from); const ColumnString::Chars_t * chars = &col_from_string->getChars(); const IColumn::Offsets * offsets = &col_from_string->getOffsets(); size_t current_offset = 0; @@ -1278,7 +1278,7 @@ struct TiDBConvertToDecimal { /// cast string as decimal const IColumn * col_from = block.getByPosition(arguments[0]).column.get(); - const ColumnString * col_from_string = checkAndGetColumn(col_from); + const auto * col_from_string = checkAndGetColumn(col_from); const ColumnString::Chars_t * chars = &col_from_string->getChars(); const IColumn::Offsets * offsets = &col_from_string->getOffsets(); size_t current_offset = 0; @@ -1415,7 +1415,7 @@ struct TiDBConvertToTime { // cast string as time const auto & col_with_type_and_name = block.getByPosition(arguments[0]); - const ColumnString * col_from = checkAndGetColumn(col_with_type_and_name.column.get()); + const auto * col_from = checkAndGetColumn(col_with_type_and_name.column.get()); const ColumnString::Chars_t * chars = &col_from->getChars(); const ColumnString::Offsets * offsets = &col_from->getOffsets(); diff --git a/dbms/src/Interpreters/WindowDescription.cpp b/dbms/src/Interpreters/WindowDescription.cpp index 4ba277605b9..31bb7356ff4 100644 --- a/dbms/src/Interpreters/WindowDescription.cpp +++ b/dbms/src/Interpreters/WindowDescription.cpp @@ -15,15 +15,17 @@ #include #include #include +#include #include +#include namespace DB { namespace ErrorCodes { extern const int BAD_ARGUMENTS; -} +} // namespace ErrorCodes WindowFrame::BoundaryType getBoundaryTypeFromTipb(const tipb::WindowFrameBound & bound) { @@ -56,9 +58,11 @@ void WindowDescription::setWindowFrame(const tipb::WindowFrame & frame_) frame.begin_offset = frame_.start().offset(); frame.begin_type = getBoundaryTypeFromTipb(frame_.start()); frame.begin_preceding = (frame_.start().type() == tipb::WindowBoundType::Preceding); + frame.begin_cmp_data_type = frame_.start().cmp_data_type(); frame.end_offset = frame_.end().offset(); frame.end_type = getBoundaryTypeFromTipb(frame_.end()); frame.end_preceding = (frame_.end().type() == tipb::WindowBoundType::Preceding); + frame.end_cmp_data_type = frame_.end().cmp_data_type(); frame.is_default = false; } diff --git a/dbms/src/Interpreters/WindowDescription.h b/dbms/src/Interpreters/WindowDescription.h index 266ee251d60..96270416bb2 100644 --- a/dbms/src/Interpreters/WindowDescription.h +++ b/dbms/src/Interpreters/WindowDescription.h @@ -63,10 +63,14 @@ struct WindowFrame BoundaryType begin_type = BoundaryType::Unbounded; UInt64 begin_offset = 0; bool begin_preceding = true; + Int32 begin_range_auxiliary_column_index = -1; + tipb::RangeCmpDataType begin_cmp_data_type; BoundaryType end_type = BoundaryType::Unbounded; UInt64 end_offset = 0; bool end_preceding = false; + Int32 end_range_auxiliary_column_index = -1; + tipb::RangeCmpDataType end_cmp_data_type; bool operator==(const WindowFrame & other) const { @@ -105,6 +109,22 @@ struct WindowDescription // The window functions that are calculated for this window. WindowFunctionDescriptions window_functions_descriptions; + // Mark the order by column type to avoid type judge + // each time we update the start/end frame position. + TypeIndex order_by_col_type = TypeIndex::Nothing; + + TypeIndex begin_aux_col_type = TypeIndex::Nothing; + TypeIndex end_aux_col_type = TypeIndex::Nothing; + + // ascending or descending for order by column + // only used for range frame type + bool is_desc; + + // only used for range frame type + bool is_order_by_col_nullable; + bool is_begin_aux_col_nullable; + bool is_end_aux_col_nullable; + void setWindowFrame(const tipb::WindowFrame & frame_); void fillArgColumnNumbers(); diff --git a/dbms/src/TestUtils/WindowTestUtils.h b/dbms/src/TestUtils/WindowTestUtils.h index 6f07ee4bafd..538d45f393e 100644 --- a/dbms/src/TestUtils/WindowTestUtils.h +++ b/dbms/src/TestUtils/WindowTestUtils.h @@ -94,13 +94,17 @@ class WindowTest : public ExecutorTest // so that caller could configure it and choose block_size and concurrency. void executeWithConcurrencyAndBlockSize( const std::shared_ptr & request, - const ColumnsWithTypeAndName & expect_columns) + const ColumnsWithTypeAndName & expect_columns, + bool is_restrict = true) { std::vector block_sizes{1, 2, 3, 4, DEFAULT_BLOCK_SIZE}; for (auto block_size : block_sizes) { context.context->setSetting("max_block_size", Field(static_cast(block_size))); - ASSERT_COLUMNS_EQ_R(expect_columns, executeStreams(request)); + if (is_restrict) + ASSERT_COLUMNS_EQ_R(expect_columns, executeStreams(request)); + else + ASSERT_COLUMNS_EQ_UR(expect_columns, executeStreams(request)); ASSERT_COLUMNS_EQ_UR(expect_columns, executeStreams(request, 2)); ASSERT_COLUMNS_EQ_UR(expect_columns, executeStreams(request, MAX_CONCURRENCY_LEVEL)); } @@ -110,7 +114,8 @@ class WindowTest : public ExecutorTest const ColumnWithTypeAndName & result, const ASTPtr & function, const ColumnsWithTypeAndName & input, - MockWindowFrame mock_frame = MockWindowFrame()) + MockWindowFrame mock_frame = MockWindowFrame(), + bool is_restrict = true) { ColumnsWithTypeAndName actual_input = input; assert(actual_input.size() == 3); @@ -135,7 +140,7 @@ class WindowTest : public ExecutorTest ColumnsWithTypeAndName expect = input; expect.push_back(result); - executeWithConcurrencyAndBlockSize(request, expect); + executeWithConcurrencyAndBlockSize(request, expect, is_restrict); } }; } // namespace DB::tests diff --git a/dbms/src/WindowFunctions/tests/gtest_first_value.cpp b/dbms/src/WindowFunctions/tests/gtest_first_value.cpp index 61b6181e457..5208245a8d2 100644 --- a/dbms/src/WindowFunctions/tests/gtest_first_value.cpp +++ b/dbms/src/WindowFunctions/tests/gtest_first_value.cpp @@ -28,6 +28,7 @@ namespace DB::tests { +// TODO support unsigned int as the test framework not supports unsigned int so far. class FirstValue : public DB::tests::WindowTest { public: @@ -70,6 +71,367 @@ class FirstValue : public DB::tests::WindowTest toNullableVec(/*order*/ {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}), toNullableVec(/*value*/ {{}, {}, 3, 4, 5, {}, 7, 8, 9, 10, {}, 12, 13})}); } + + void testIntOrderByColForRangeFrame() + { + MockWindowFrame mock_frame; + mock_frame.type = tipb::WindowFrameType::Ranges; + mock_frame.start = mock::MockWindowFrameBound(tipb::WindowBoundType::Preceding, false, 0); + mock_frame.end = buildRangeFrameBound( + tipb::WindowBoundType::Following, + tipb::RangeCmpDataType::Int, + ORDER_COL_NAME, + true, + static_cast(0)); + + { + // Int type const column + std::vector frame_start_range{0, 1, 3, 10}; + std::vector>> res{ + {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}, + {0, 1, 1, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}, + {0, 1, 1, 1, 8, 0, 0, 10, 10, 13, 1, 1, 3, 9, 15, 20, 31}, + {0, 1, 1, 1, 1, 0, 0, 0, 3, 10, 1, 1, 1, 1, 5, 15, 31}}; + + auto partition_col = toVec(/*partition*/ {0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3}); + auto order_col = toVec(/*order*/ {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}); + auto val_col = toVec(/*value*/ {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}); + + for (size_t i = 0; i < frame_start_range.size(); ++i) + { + mock_frame.start = buildRangeFrameBound( + tipb::WindowBoundType::Preceding, + tipb::RangeCmpDataType::Int, + ORDER_COL_NAME, + false, + frame_start_range[i]); + executeFunctionAndAssert( + toNullableVec(res[i]), + FirstValue(value_col), + {partition_col, order_col, val_col}, + mock_frame); + } + } + + { + // Float type const column + std::vector frame_start_range{0, 1.1, 2.9, 9.9}; + std::vector>> res{ + {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}, + {0, 1, 1, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}, + {0, 1, 1, 2, 8, 0, 3, 10, 13, 13, 1, 1, 3, 9, 15, 20, 31}, + {0, 1, 1, 1, 1, 0, 0, 3, 10, 10, 1, 1, 1, 1, 9, 15, 31}}; + + auto partition_col = toVec(/*partition*/ {0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3}); + auto order_col = toVec(/*order*/ {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}); + auto val_col = toVec(/*value*/ {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}); + + for (size_t i = 0; i < frame_start_range.size(); ++i) + { + mock_frame.start = buildRangeFrameBound( + tipb::WindowBoundType::Preceding, + tipb::RangeCmpDataType::Float, + ORDER_COL_NAME, + false, + frame_start_range[i]); + executeFunctionAndAssert( + toNullableVec(res[i]), + FirstValue(value_col), + {partition_col, order_col, val_col}, + mock_frame); + } + } + + { + // Decimal type const column + std::vector> frame_start_range{ + DecimalField(0, 0), + DecimalField(11, 1), + DecimalField(29, 1), + DecimalField(99, 1)}; + std::vector>> res{ + {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}, + {0, 1, 1, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}, + {0, 1, 1, 2, 8, 0, 3, 10, 13, 13, 1, 1, 3, 9, 15, 20, 31}, + {0, 1, 1, 1, 1, 0, 0, 3, 10, 10, 1, 1, 1, 1, 9, 15, 31}}; + + auto partition_col = toVec(/*partition*/ {0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3}); + auto order_col = toVec(/*order*/ {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}); + auto val_col = toVec(/*value*/ {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}); + + for (size_t i = 0; i < frame_start_range.size(); ++i) + { + mock_frame.start = buildRangeFrameBound( + tipb::WindowBoundType::Preceding, + tipb::RangeCmpDataType::Decimal, + ORDER_COL_NAME, + false, + frame_start_range[i]); + executeFunctionAndAssert( + toNullableVec(res[i]), + FirstValue(value_col), + {partition_col, order_col, val_col}, + mock_frame); + } + } + } + + void testFloatOrderByColForRangeFrame() + { + MockWindowFrame mock_frame; + mock_frame.type = tipb::WindowFrameType::Ranges; + mock_frame.start = mock::MockWindowFrameBound(tipb::WindowBoundType::Preceding, false, 0); + mock_frame.end = buildRangeFrameBound( + tipb::WindowBoundType::Following, + tipb::RangeCmpDataType::Float, + ORDER_COL_NAME, + true, + static_cast(0)); + + { + // Int type const column + std::vector frame_start_range{0, 1, 3, 10}; + std::vector>> res{ + {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}, + {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}, + {0, 1, 1, 2, 8, 0, 3, 10, 13, 13, 1, 1, 3, 9, 15, 20, 31}, + {0, 1, 1, 1, 1, 0, 0, 0, 3, 10, 1, 1, 1, 1, 5, 15, 31}}; + + auto partition_col = toVec(/*partition*/ {0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3}); + auto order_col = toVec( + /*order*/ {0.1, 1.0, 2.1, 4.1, 8.1, 0.0, 3.1, 10.0, 13.1, 15.1, 1.1, 2.9, 5.1, 9.1, 15.0, 20.1, 31.1}); + auto val_col = toVec(/*value*/ {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}); + + for (size_t i = 0; i < frame_start_range.size(); ++i) + { + mock_frame.start = buildRangeFrameBound( + tipb::WindowBoundType::Preceding, + tipb::RangeCmpDataType::Float, + ORDER_COL_NAME, + false, + frame_start_range[i]); + executeFunctionAndAssert( + toNullableVec(res[i]), + FirstValue(value_col), + {partition_col, order_col, val_col}, + mock_frame); + } + } + + { + // Float type const column + std::vector frame_start_range{0, 1.8, 2.3, 3.8}; + std::vector>> res{ + {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}, + {0, 1, 1, 4, 8, 0, 3, 10, 13, 15, 1, 1, 5, 9, 15, 20, 31}, + {0, 1, 1, 2, 8, 0, 3, 10, 13, 13, 1, 1, 3, 9, 15, 20, 31}, + {0, 1, 1, 1, 8, 0, 0, 10, 10, 13, 1, 1, 3, 9, 15, 20, 31}}; + + auto partition_col = toVec(/*partition*/ {0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3}); + auto order_col = toVec( + /*order*/ {0.1, 1.0, 2.1, 4.1, 8.1, 0.0, 3.1, 10.0, 13.1, 15.1, 1.1, 2.9, 5.1, 9.1, 15.0, 20.1, 31.1}); + auto val_col = toVec(/*value*/ {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}); + + for (size_t i = 0; i < frame_start_range.size(); ++i) + { + mock_frame.start = buildRangeFrameBound( + tipb::WindowBoundType::Preceding, + tipb::RangeCmpDataType::Float, + ORDER_COL_NAME, + false, + frame_start_range[i]); + executeFunctionAndAssert( + toNullableVec(res[i]), + FirstValue(value_col), + {partition_col, order_col, val_col}, + mock_frame); + } + } + + { + // Decimal type const column + std::vector> frame_start_range{ + DecimalField(0, 0), + DecimalField(18, 1), + DecimalField(23, 1), + DecimalField(38, 1)}; + std::vector>> res{ + {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}, + {0, 1, 1, 4, 8, 0, 3, 10, 13, 15, 1, 1, 5, 9, 15, 20, 31}, + {0, 1, 1, 2, 8, 0, 3, 10, 13, 13, 1, 1, 3, 9, 15, 20, 31}, + {0, 1, 1, 1, 8, 0, 0, 10, 10, 13, 1, 1, 3, 9, 15, 20, 31}}; + + auto partition_col = toVec(/*partition*/ {0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3}); + auto order_col = toVec( + /*order*/ {0.1, 1.0, 2.1, 4.1, 8.1, 0.0, 3.1, 10.0, 13.1, 15.1, 1.1, 2.9, 5.1, 9.1, 15.0, 20.1, 31.1}); + auto val_col = toVec(/*value*/ {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}); + + for (size_t i = 0; i < frame_start_range.size(); ++i) + { + mock_frame.start = buildRangeFrameBound( + tipb::WindowBoundType::Preceding, + tipb::RangeCmpDataType::Float, + ORDER_COL_NAME, + false, + frame_start_range[i]); + executeFunctionAndAssert( + toNullableVec(res[i]), + FirstValue(value_col), + {partition_col, order_col, val_col}, + mock_frame); + } + } + } + + void testDecimalOrderByColForRangeFrame() + { + MockWindowFrame mock_frame; + mock_frame.type = tipb::WindowFrameType::Ranges; + mock_frame.start = mock::MockWindowFrameBound(tipb::WindowBoundType::Preceding, false, 0); + mock_frame.end = buildRangeFrameBound( + tipb::WindowBoundType::Following, + tipb::RangeCmpDataType::Decimal, + ORDER_COL_NAME, + true, + static_cast(0)); + + { + // Int type const column + std::vector frame_start_range{0, 1, 3, 10}; + std::vector>> res{ + {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}, + {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}, + {0, 1, 1, 2, 8, 0, 3, 10, 13, 13, 1, 1, 3, 9, 15, 20, 31}, + {0, 1, 1, 1, 1, 0, 0, 0, 3, 10, 1, 1, 1, 1, 5, 15, 31}}; + + auto partition_col = toVec(/*partition*/ {0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3}); + auto order_col = createColumn(/*order*/ + std::make_tuple(3, 1), + { + DecimalField(1, 1), + DecimalField(10, 1), + DecimalField(21, 1), + DecimalField(41, 1), + DecimalField(81, 1), + DecimalField(0, 1), + DecimalField(31, 1), + DecimalField(100, 1), + DecimalField(131, 1), + DecimalField(151, 1), + DecimalField(11, 1), + DecimalField(29, 1), + DecimalField(51, 1), + DecimalField(91, 1), + DecimalField(150, 1), + DecimalField(201, 1), + DecimalField(311, 1), + }); + auto val_col = toVec(/*value*/ {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}); + + for (size_t i = 0; i < frame_start_range.size(); ++i) + { + mock_frame.start = buildRangeFrameBound( + tipb::WindowBoundType::Preceding, + tipb::RangeCmpDataType::Decimal, + ORDER_COL_NAME, + false, + frame_start_range[i]); + executeFunctionAndAssert( + toNullableVec(res[i]), + FirstValue(value_col), + {partition_col, order_col, val_col}, + mock_frame); + } + } + + // TODO Support Float type const column + // TODO Support Decimal type const column + } + + void testNullableOrderByColForRangeFrame() + { + MockWindowFrame mock_frame; + mock_frame.type = tipb::WindowFrameType::Ranges; + mock_frame.start = mock::MockWindowFrameBound(tipb::WindowBoundType::Preceding, false, 0); + mock_frame.end = buildRangeFrameBound( + tipb::WindowBoundType::Following, + tipb::RangeCmpDataType::Int, + ORDER_COL_NAME, + true, + static_cast(0)); + + auto partition_col = toVec(/*partition*/ {0, 1, 1, 1, 2, 2, 2, 2, 2}); + auto order_col = toNullableVec(/*order*/ {0, {}, 1, 2, {}, 5, 6, 9, 10}); + auto val_col = toVec(/*value*/ {1, 2, 3, 4, 5, 6, 7, 8, 9}); + + { + std::vector frame_start_range{0, 1, 3, 10}; + std::vector>> res{ + {1, 2, 3, 4, 5, 6, 7, 8, 9}, + {1, 2, 3, 3, 5, 6, 6, 8, 8}, + {1, 2, 3, 3, 5, 6, 6, 7, 8}, + {1, 2, 3, 3, 5, 6, 6, 6, 6}}; + + for (size_t i = 0; i < frame_start_range.size(); ++i) + { + mock_frame.start = buildRangeFrameBound( + tipb::WindowBoundType::Preceding, + tipb::RangeCmpDataType::Int, + ORDER_COL_NAME, + false, + frame_start_range[i]); + executeFunctionAndAssert( + toNullableVec(res[i]), + FirstValue(value_col), + {partition_col, order_col, val_col}, + mock_frame); + } + } + + { + // + mock_frame.start = buildRangeFrameBound( + tipb::WindowBoundType::Preceding, + tipb::RangeCmpDataType::Float, + ORDER_COL_NAME, + false, + static_cast(1)); + mock_frame.end = buildRangeFrameBound( + tipb::WindowBoundType::Preceding, + tipb::RangeCmpDataType::Float, + ORDER_COL_NAME, + false, + static_cast(1)); + + executeFunctionAndAssert( + toNullableVec({{}, 2, {}, 3, 5, {}, 6, {}, 8}), + FirstValue(value_col), + {partition_col, order_col, val_col}, + mock_frame); + } + + { + // + mock_frame.start = buildRangeFrameBound( + tipb::WindowBoundType::Following, + tipb::RangeCmpDataType::Float, + ORDER_COL_NAME, + true, + static_cast(1)); + mock_frame.end = buildRangeFrameBound( + tipb::WindowBoundType::Following, + tipb::RangeCmpDataType::Float, + ORDER_COL_NAME, + true, + static_cast(1)); + + executeFunctionAndAssert( + toNullableVec({{}, 2, 4, {}, 5, 7, {}, 9, {}}), + FirstValue(value_col), + {partition_col, order_col, val_col}, + mock_frame); + } + } }; TEST_F(FirstValue, firstValueWithRowsFrameType) @@ -154,7 +516,6 @@ try frame); } - // TODO support unsigned int. testInt(); testInt(); testInt(); @@ -165,4 +526,20 @@ try } CATCH +// This is the test just for testing range type frame. +// Not every window function needs this test. +TEST_F(FirstValue, firstValueWithRangeFrameType) +try +{ + testIntOrderByColForRangeFrame(); + testFloatOrderByColForRangeFrame(); + testNullableOrderByColForRangeFrame(); + + // TODO This test is disabled as we can not assign decimal field's flen now + // in MockStorage.cpp::mockColumnInfosToTiDBColumnInfos(). + // However, we will test this data type in fullstack tests. + // testDecimalOrderByColForRangeFrame(); +} +CATCH + } // namespace DB::tests diff --git a/dbms/src/WindowFunctions/tests/gtest_last_value.cpp b/dbms/src/WindowFunctions/tests/gtest_last_value.cpp index 37a6edc924b..cc60b5c7c52 100644 --- a/dbms/src/WindowFunctions/tests/gtest_last_value.cpp +++ b/dbms/src/WindowFunctions/tests/gtest_last_value.cpp @@ -23,6 +23,7 @@ namespace DB::tests { +// TODO support unsigned int as the test framework not supports unsigned int so far. class LastValue : public DB::tests::WindowTest { public: @@ -79,6 +80,315 @@ class LastValue : public DB::tests::WindowTest toNullableVec(/*value*/ {{}, 2, 3, 4, {}, 6, 7, 8, 9, {}, 11, 12, {}})}, unbounded_type_frame); } + + void testIntOrderByColForRangeFrame() + { + MockWindowFrame mock_frame; + mock_frame.type = tipb::WindowFrameType::Ranges; + mock_frame.start = buildRangeFrameBound( + tipb::WindowBoundType::Preceding, + tipb::RangeCmpDataType::Float, + ORDER_COL_NAME, + false, + static_cast(0)); + mock_frame.end = mock::MockWindowFrameBound(tipb::WindowBoundType::Following, false, 0); + + { + // Int type const column + std::vector frame_end_range{0, 1, 3, 10}; + std::vector>> res{ + {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}, + {0, 2, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}, + {0, 4, 4, 4, 8, 3, 3, 13, 15, 15, 3, 5, 5, 9, 15, 20, 31}, + {0, 8, 8, 8, 8, 10, 13, 15, 15, 15, 9, 9, 15, 15, 20, 20, 31}}; + + auto partition_col = toVec(/*partition*/ {0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3}); + auto order_col = toVec(/*order*/ {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}); + auto val_col = toVec(/*value*/ {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}); + + for (size_t i = 0; i < frame_end_range.size(); ++i) + { + mock_frame.end = buildRangeFrameBound( + tipb::WindowBoundType::Following, + tipb::RangeCmpDataType::Int, + ORDER_COL_NAME, + true, + frame_end_range[i]); + executeFunctionAndAssert( + toNullableVec(res[i]), + LastValue(value_col), + {partition_col, order_col, val_col}, + mock_frame); + } + } + + { + // Float type const column + std::vector frame_start_range{0, 1.1, 2.9, 9.9}; + std::vector>> res{ + {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}, + {0, 2, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}, + {0, 2, 4, 4, 8, 0, 3, 10, 15, 15, 3, 5, 5, 9, 15, 20, 31}, + {0, 8, 8, 8, 8, 3, 10, 15, 15, 15, 9, 9, 9, 15, 20, 20, 31}}; + + auto partition_col = toVec(/*partition*/ {0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3}); + auto order_col = toVec(/*order*/ {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}); + auto val_col = toVec(/*value*/ {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}); + + for (size_t i = 0; i < frame_start_range.size(); ++i) + { + mock_frame.end = buildRangeFrameBound( + tipb::WindowBoundType::Following, + tipb::RangeCmpDataType::Float, + ORDER_COL_NAME, + true, + frame_start_range[i]); + executeFunctionAndAssert( + toNullableVec(res[i]), + LastValue(value_col), + {partition_col, order_col, val_col}, + mock_frame); + } + } + + { + // Decimal type const column + std::vector> frame_start_range{ + DecimalField(0, 0), + DecimalField(11, 1), + DecimalField(29, 1), + DecimalField(99, 1)}; + std::vector>> res{ + {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}, + {0, 2, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}, + {0, 2, 4, 4, 8, 0, 3, 10, 15, 15, 3, 5, 5, 9, 15, 20, 31}, + {0, 8, 8, 8, 8, 3, 10, 15, 15, 15, 9, 9, 9, 15, 20, 20, 31}}; + + auto partition_col = toVec(/*partition*/ {0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3}); + auto order_col = toVec(/*order*/ {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}); + auto val_col = toVec(/*value*/ {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}); + + for (size_t i = 0; i < frame_start_range.size(); ++i) + { + mock_frame.end = buildRangeFrameBound( + tipb::WindowBoundType::Following, + tipb::RangeCmpDataType::Decimal, + ORDER_COL_NAME, + true, + frame_start_range[i]); + executeFunctionAndAssert( + toNullableVec(res[i]), + LastValue(value_col), + {partition_col, order_col, val_col}, + mock_frame); + } + } + } + + void testFloatOrderByColForRangeFrame() + { + MockWindowFrame mock_frame; + mock_frame.type = tipb::WindowFrameType::Ranges; + mock_frame.start = buildRangeFrameBound( + tipb::WindowBoundType::Preceding, + tipb::RangeCmpDataType::Float, + ORDER_COL_NAME, + false, + static_cast(0)); + mock_frame.end = mock::MockWindowFrameBound(tipb::WindowBoundType::Following, false, static_cast(0)); + + { + // Int type const column + std::vector frame_start_range{0, 1, 3, 10}; + std::vector>> res{ + {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}, + {0, 2, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}, + {0, 4, 4, 4, 8, 0, 3, 10, 15, 15, 3, 5, 5, 9, 15, 20, 31}, + {0, 8, 8, 8, 8, 10, 13, 15, 15, 15, 9, 9, 15, 15, 20, 20, 31}}; + + auto partition_col = toVec(/*partition*/ {0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3}); + auto order_col = toVec( + /*order*/ {0.1, 1.2, 2.1, 4.1, 8.1, 0.0, 3.1, 10.0, 13.1, 15.1, 1.1, 2.9, 5.1, 9.1, 15.0, 20.1, 31.1}); + auto val_col = toVec(/*value*/ {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}); + + for (size_t i = 0; i < frame_start_range.size(); ++i) + { + mock_frame.end = buildRangeFrameBound( + tipb::WindowBoundType::Following, + tipb::RangeCmpDataType::Float, + ORDER_COL_NAME, + true, + frame_start_range[i]); + executeFunctionAndAssert( + toNullableVec(res[i]), + LastValue(value_col), + {partition_col, order_col, val_col}, + mock_frame); + } + } + + { + // Float type const column + std::vector frame_start_range{0, 1.1, 2.3, 3.8}; + std::vector>> res{ + {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}, + {0, 2, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}, + {0, 2, 4, 4, 8, 0, 3, 10, 15, 15, 3, 5, 5, 9, 15, 20, 31}, + {0, 4, 4, 4, 8, 3, 3, 13, 15, 15, 3, 5, 5, 9, 15, 20, 31}}; + + auto partition_col = toVec(/*partition*/ {0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3}); + auto order_col = toVec( + /*order*/ {0.1, 1.2, 2.1, 4.1, 8.1, 0.0, 3.1, 10.0, 13.1, 15.1, 1.1, 2.9, 5.1, 9.1, 15.0, 20.1, 31.1}); + auto val_col = toVec(/*value*/ {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}); + + for (size_t i = 0; i < frame_start_range.size(); ++i) + { + mock_frame.end = buildRangeFrameBound( + tipb::WindowBoundType::Following, + tipb::RangeCmpDataType::Float, + ORDER_COL_NAME, + true, + frame_start_range[i]); + executeFunctionAndAssert( + toNullableVec(res[i]), + LastValue(value_col), + {partition_col, order_col, val_col}, + mock_frame); + } + } + + { + // Decimal type const column + std::vector> frame_start_range{ + DecimalField(0, 0), + DecimalField(11, 1), + DecimalField(23, 1), + DecimalField(38, 1)}; + std::vector>> res{ + {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}, + {0, 2, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}, + {0, 2, 4, 4, 8, 0, 3, 10, 15, 15, 3, 5, 5, 9, 15, 20, 31}, + {0, 4, 4, 4, 8, 3, 3, 13, 15, 15, 3, 5, 5, 9, 15, 20, 31}}; + + auto partition_col = toVec(/*partition*/ {0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3}); + auto order_col = toVec( + /*order*/ {0.1, 1.2, 2.1, 4.1, 8.1, 0.0, 3.1, 10.0, 13.1, 15.1, 1.1, 2.9, 5.1, 9.1, 15.0, 20.1, 31.1}); + auto val_col = toVec(/*value*/ {0, 1, 2, 4, 8, 0, 3, 10, 13, 15, 1, 3, 5, 9, 15, 20, 31}); + + for (size_t i = 0; i < frame_start_range.size(); ++i) + { + mock_frame.end = buildRangeFrameBound( + tipb::WindowBoundType::Following, + tipb::RangeCmpDataType::Float, + ORDER_COL_NAME, + true, + frame_start_range[i]); + executeFunctionAndAssert( + toNullableVec(res[i]), + LastValue(value_col), + {partition_col, order_col, val_col}, + mock_frame); + } + } + } + + void testNullableOrderByColForRangeFrame() + { + MockWindowFrame mock_frame; + mock_frame.type = tipb::WindowFrameType::Ranges; + mock_frame.start = buildRangeFrameBound( + tipb::WindowBoundType::Preceding, + tipb::RangeCmpDataType::Int, + ORDER_COL_NAME, + false, + static_cast(0)); + ; + mock_frame.end = buildRangeFrameBound( + tipb::WindowBoundType::Following, + tipb::RangeCmpDataType::Int, + ORDER_COL_NAME, + true, + static_cast(0)); + + auto partition_col = toVec(/*partition*/ {0, 1, 1, 1, 2, 2, 2, 2, 2}); + auto order_col = toNullableVec(/*order*/ {0, {}, 1, 2, {}, 5, 6, 9, 10}); + auto val_col = toVec(/*value*/ {1, 2, 3, 4, 5, 6, 7, 8, 9}); + + { + std::vector frame_end_range{0, 1, 3, 10}; + std::vector>> res{ + {1, 2, 3, 4, 5, 6, 7, 8, 9}, + {1, 2, 4, 4, 5, 7, 7, 9, 9}, + {1, 2, 4, 4, 5, 7, 8, 9, 9}, + {1, 2, 4, 4, 5, 9, 9, 9, 9}}; + + for (size_t i = 0; i < frame_end_range.size(); ++i) + { + mock_frame.end = buildRangeFrameBound( + tipb::WindowBoundType::Following, + tipb::RangeCmpDataType::Int, + ORDER_COL_NAME, + true, + frame_end_range[i]); + executeFunctionAndAssert( + toNullableVec(res[i]), + LastValue(value_col), + {partition_col, order_col, val_col}, + mock_frame); + } + } + + { + // + mock_frame.start = buildRangeFrameBound( + tipb::WindowBoundType::Preceding, + tipb::RangeCmpDataType::Float, + ORDER_COL_NAME, + false, + static_cast(1)); + mock_frame.end = buildRangeFrameBound( + tipb::WindowBoundType::Preceding, + tipb::RangeCmpDataType::Float, + ORDER_COL_NAME, + false, + static_cast(1)); + + executeFunctionAndAssert( + toNullableVec({{}, 2, {}, 3, 5, {}, 6, {}, 8}), + LastValue(value_col), + {partition_col, order_col, val_col}, + mock_frame); + } + + { + // + mock_frame.start = buildRangeFrameBound( + tipb::WindowBoundType::Following, + tipb::RangeCmpDataType::Float, + ORDER_COL_NAME, + true, + static_cast(1)); + mock_frame.end = buildRangeFrameBound( + tipb::WindowBoundType::Following, + tipb::RangeCmpDataType::Float, + ORDER_COL_NAME, + true, + static_cast(1)); + + executeFunctionAndAssert( + toNullableVec({{}, 2, 4, {}, 5, 7, {}, 9, {}}), + LastValue(value_col), + {partition_col, order_col, val_col}, + mock_frame); + } + } + + void testDecimalOrderByColForRangeFrame() + { + // TODO we can not assign decimal field's flen now + // in MockStorage.cpp::mockColumnInfosToTiDBColumnInfos(). + // However, we will test this data type in fullstack tests. + } }; TEST_F(LastValue, lastValue) @@ -164,7 +474,6 @@ try frame); } - // TODO support unsigned int. testInt(); testInt(); testInt(); @@ -175,4 +484,16 @@ try } CATCH +// This is the test just for testing range type frame. +// Not every window function needs this test. +TEST_F(LastValue, lastValueWithRangeFrameType) +try +{ + testIntOrderByColForRangeFrame(); + testFloatOrderByColForRangeFrame(); + testNullableOrderByColForRangeFrame(); + // TODO Implement testDecimalOrderByColForRangeFrame() +} +CATCH + } // namespace DB::tests diff --git a/dbms/src/WindowFunctions/tests/gtest_lead_lag.cpp b/dbms/src/WindowFunctions/tests/gtest_lead_lag.cpp index 48c164ae714..85d9d79dc26 100644 --- a/dbms/src/WindowFunctions/tests/gtest_lead_lag.cpp +++ b/dbms/src/WindowFunctions/tests/gtest_lead_lag.cpp @@ -22,8 +22,7 @@ namespace DB::tests template using Limits = std::numeric_limits; -// TODO Support more convenient testing framework for Window Function. -// TODO Tests with frame should be added +// TODO support unsigned int as the test framework not supports unsigned int so far. class LeadLag : public DB::tests::WindowTest { public: @@ -367,8 +366,6 @@ CATCH TEST_F(LeadLag, Int) try { - // TODO support unsigned int. - testInt(); testInt(); testInt(); diff --git a/tests/fullstack-test/mpp/window_range_frame.test b/tests/fullstack-test/mpp/window_range_frame.test new file mode 100644 index 00000000000..31086dc80d0 --- /dev/null +++ b/tests/fullstack-test/mpp/window_range_frame.test @@ -0,0 +1,957 @@ +# Copyright 2023 PingCAP, Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# first_value function tests the preceding range +# last_value function tests the following range + +# first_value tests +mysql> drop table if exists test.first_range; +mysql> create table test.first_range(p int not null, o int not null, v int not null); +mysql> insert into test.first_range (p, o, v) values (0, 0, 0), (1, 1, 1), (1, 2, 2), (1, 4, 4), (1, 8, 8), (2, 0, 0), (2, 3, 3), (2, 10, 10), (2, 13, 13), (2, 15, 15), (3, 1, 1), (3, 3, 3), (3, 5, 5), (3, 9, 9), (3, 15, 15), (3, 20, 20), (3, 31, 31); +mysql> alter table test.first_range set tiflash replica 1; + +mysql> drop table if exists test.first_range_f; +mysql> create table test.first_range_f(p int not null, o double not null, v int not null); +mysql> insert into test.first_range_f (p, o, v) values (0, 0.1, 0), (1, 1.0, 1), (1, 2.1, 2), (1, 4.1, 4), (1, 8.1, 8), (2, 0.0, 0), (2, 3.1, 3), (2, 10.0, 10), (2, 13.1, 13), (2, 15.1, 15), (3, 1.1, 1), (3, 2.9, 3), (3, 5.1, 5), (3, 9.1, 9), (3, 15.0, 15), (3, 20.1, 20), (3, 31.1, 31); +mysql> alter table test.first_range_f set tiflash replica 1; + +mysql> drop table if exists test.first_range_d32; +mysql> create table test.first_range_d32(p int not null, o decimal(7,1) not null, v int not null); +mysql> insert into test.first_range_d32 (p, o, v) values (0, 0.1, 0), (1, 1.0, 1), (1, 2.1, 2), (1, 4.1, 4), (1, 8.1, 8), (2, 0.0, 0), (2, 3.1, 3), (2, 10.0, 10), (2, 13.1, 13), (2, 15.1, 15), (3, 1.1, 1), (3, 2.9, 3), (3, 5.1, 5), (3, 9.1, 9), (3, 15.0, 15), (3, 20.1, 20), (3, 31.1, 31); +mysql> alter table test.first_range_d32 set tiflash replica 1; + +mysql> drop table if exists test.first_range_d64; +mysql> create table test.first_range_d64(p int not null, o decimal(17,1) not null, v int not null); +mysql> insert into test.first_range_d64 (p, o, v) values (0, 0.1, 0), (1, 1.0, 1), (1, 2.1, 2), (1, 4.1, 4), (1, 8.1, 8), (2, 0.0, 0), (2, 3.1, 3), (2, 10.0, 10), (2, 13.1, 13), (2, 15.1, 15), (3, 1.1, 1), (3, 2.9, 3), (3, 5.1, 5), (3, 9.1, 9), (3, 15.0, 15), (3, 20.1, 20), (3, 31.1, 31); +mysql> alter table test.first_range_d64 set tiflash replica 1; + +mysql> drop table if exists test.first_range_d128; +mysql> create table test.first_range_d128(p int not null, o decimal(37,1) not null, v int not null); +mysql> insert into test.first_range_d128 (p, o, v) values (0, 0.1, 0), (1, 1.0, 1), (1, 2.1, 2), (1, 4.1, 4), (1, 8.1, 8), (2, 0.0, 0), (2, 3.1, 3), (2, 10.0, 10), (2, 13.1, 13), (2, 15.1, 15), (3, 1.1, 1), (3, 2.9, 3), (3, 5.1, 5), (3, 9.1, 9), (3, 15.0, 15), (3, 20.1, 20), (3, 31.1, 31); +mysql> alter table test.first_range_d128 set tiflash replica 1; + +mysql> drop table if exists test.first_range_d256; +mysql> create table test.first_range_d256(p int not null, o decimal(64,1) not null, v int not null); +mysql> insert into test.first_range_d256 (p, o, v) values (0, 0.1, 0), (1, 1.0, 1), (1, 2.1, 2), (1, 4.1, 4), (1, 8.1, 8), (2, 0.0, 0), (2, 3.1, 3), (2, 10.0, 10), (2, 13.1, 13), (2, 15.1, 15), (3, 1.1, 1), (3, 2.9, 3), (3, 5.1, 5), (3, 9.1, 9), (3, 15.0, 15), (3, 20.1, 20), (3, 31.1, 31); +mysql> alter table test.first_range_d256 set tiflash replica 1; + +func> wait_table test first_range +func> wait_table test first_range_f +func> wait_table test first_range_d32 +func> wait_table test first_range_d64 +func> wait_table test first_range_d128 +func> wait_table test first_range_d256 + +mysql> use test; set tidb_enforce_mpp=1; select *, first_value(v) over (partition by p order by o range between 3 preceding and 0 following) as a from test.first_range; ++---+----+----+----+ +| p | o | v | a | ++---+----+----+----+ +| 1 | 1 | 1 | 1 | +| 1 | 2 | 2 | 1 | +| 1 | 4 | 4 | 1 | +| 1 | 8 | 8 | 8 | +| 2 | 0 | 0 | 0 | +| 2 | 3 | 3 | 0 | +| 2 | 10 | 10 | 10 | +| 2 | 13 | 13 | 10 | +| 2 | 15 | 15 | 13 | +| 3 | 1 | 1 | 1 | +| 3 | 3 | 3 | 1 | +| 3 | 5 | 5 | 3 | +| 3 | 9 | 9 | 9 | +| 3 | 15 | 15 | 15 | +| 3 | 20 | 20 | 20 | +| 3 | 31 | 31 | 31 | +| 0 | 0 | 0 | 0 | ++---+----+----+----+ + +mysql> use test; set tidb_enforce_mpp=1; select *, first_value(v) over (partition by p order by o range between 3 preceding and 2.9E0 following) as a from test.first_range; ++---+----+----+----+ +| p | o | v | a | ++---+----+----+----+ +| 0 | 0 | 0 | 0 | +| 3 | 1 | 1 | 1 | +| 3 | 3 | 3 | 1 | +| 3 | 5 | 5 | 3 | +| 3 | 9 | 9 | 9 | +| 3 | 15 | 15 | 15 | +| 3 | 20 | 20 | 20 | +| 3 | 31 | 31 | 31 | +| 2 | 0 | 0 | 0 | +| 2 | 3 | 3 | 0 | +| 2 | 10 | 10 | 10 | +| 2 | 13 | 13 | 10 | +| 2 | 15 | 15 | 13 | +| 1 | 1 | 1 | 1 | +| 1 | 2 | 2 | 1 | +| 1 | 4 | 4 | 1 | +| 1 | 8 | 8 | 8 | ++---+----+----+----+ + +# TODO Uncomment this after tidb bug is adressed +# mysql> use test; set tidb_enforce_mpp=1; select *, first_value(v) over (partition by p order by o range between 3 preceding and 2.9 following) as a from test.first_range; +# +---+----+----+----+ +# | p | o | v | a | +# +---+----+----+----+ +# | 0 | 0 | 0 | 0 | +# | 1 | 1 | 1 | 1 | +# | 1 | 2 | 2 | 1 | +# | 1 | 4 | 4 | 1 | +# | 1 | 8 | 8 | 8 | +# | 2 | 0 | 0 | 0 | +# | 2 | 3 | 3 | 0 | +# | 2 | 10 | 10 | 10 | +# | 2 | 13 | 13 | 10 | +# | 2 | 15 | 15 | 13 | +# | 3 | 1 | 1 | 1 | +# | 3 | 3 | 3 | 1 | +# | 3 | 5 | 5 | 3 | +# | 3 | 9 | 9 | 9 | +# | 3 | 15 | 15 | 15 | +# | 3 | 20 | 20 | 20 | +# | 3 | 31 | 31 | 31 | +# +---+----+----+----+ + +mysql> use test; set tidb_enforce_mpp=1; select *, first_value(v) over (partition by p order by o range between 3 preceding and 0 following) as a from test.first_range_f; ++---+------+----+----+ +| p | o | v | a | ++---+------+----+----+ +| 1 | 1 | 1 | 1 | +| 1 | 2.1 | 2 | 1 | +| 1 | 4.1 | 4 | 2 | +| 1 | 8.1 | 8 | 8 | +| 3 | 1.1 | 1 | 1 | +| 3 | 2.9 | 3 | 1 | +| 3 | 5.1 | 5 | 3 | +| 3 | 9.1 | 9 | 9 | +| 3 | 15 | 15 | 15 | +| 3 | 20.1 | 20 | 20 | +| 3 | 31.1 | 31 | 31 | +| 2 | 0 | 0 | 0 | +| 2 | 3.1 | 3 | 3 | +| 2 | 10 | 10 | 10 | +| 2 | 13.1 | 13 | 13 | +| 2 | 15.1 | 15 | 13 | +| 0 | 0.1 | 0 | 0 | ++---+------+----+----+ + +mysql> use test; set tidb_enforce_mpp=1; select *, first_value(v) over (partition by p order by o range between 2.3E0 preceding and 0 following) as a from test.first_range_f; ++---+------+----+----+ +| p | o | v | a | ++---+------+----+----+ +| 0 | 0.1 | 0 | 0 | +| 3 | 1.1 | 1 | 1 | +| 3 | 2.9 | 3 | 1 | +| 3 | 5.1 | 5 | 3 | +| 3 | 9.1 | 9 | 9 | +| 3 | 15 | 15 | 15 | +| 3 | 20.1 | 20 | 20 | +| 3 | 31.1 | 31 | 31 | +| 2 | 0 | 0 | 0 | +| 2 | 3.1 | 3 | 3 | +| 2 | 10 | 10 | 10 | +| 2 | 13.1 | 13 | 13 | +| 2 | 15.1 | 15 | 13 | +| 1 | 1 | 1 | 1 | +| 1 | 2.1 | 2 | 1 | +| 1 | 4.1 | 4 | 2 | +| 1 | 8.1 | 8 | 8 | ++---+------+----+----+ + +mysql> use test; set tidb_enforce_mpp=1; select *, first_value(v) over (partition by p order by o range between 2.3 preceding and 0 following) as a from test.first_range_f; ++---+------+----+----+ +| p | o | v | a | ++---+------+----+----+ +| 1 | 1 | 1 | 1 | +| 1 | 2.1 | 2 | 1 | +| 1 | 4.1 | 4 | 2 | +| 1 | 8.1 | 8 | 8 | +| 3 | 1.1 | 1 | 1 | +| 3 | 2.9 | 3 | 1 | +| 3 | 5.1 | 5 | 3 | +| 3 | 9.1 | 9 | 9 | +| 3 | 15 | 15 | 15 | +| 3 | 20.1 | 20 | 20 | +| 3 | 31.1 | 31 | 31 | +| 2 | 0 | 0 | 0 | +| 2 | 3.1 | 3 | 3 | +| 2 | 10 | 10 | 10 | +| 2 | 13.1 | 13 | 13 | +| 2 | 15.1 | 15 | 13 | +| 0 | 0.1 | 0 | 0 | ++---+------+----+----+ + +mysql> use test; set tidb_enforce_mpp=1; select *, first_value(v) over (partition by p order by o range between 3 preceding and 0 following) as a from test.first_range_d32; ++---+------+----+----+ +| p | o | v | a | ++---+------+----+----+ +| 3 | 1.1 | 1 | 1 | +| 3 | 2.9 | 3 | 1 | +| 3 | 5.1 | 5 | 3 | +| 3 | 9.1 | 9 | 9 | +| 3 | 15.0 | 15 | 15 | +| 3 | 20.1 | 20 | 20 | +| 3 | 31.1 | 31 | 31 | +| 2 | 0.0 | 0 | 0 | +| 2 | 3.1 | 3 | 3 | +| 2 | 10.0 | 10 | 10 | +| 2 | 13.1 | 13 | 13 | +| 2 | 15.1 | 15 | 13 | +| 0 | 0.1 | 0 | 0 | +| 1 | 1.0 | 1 | 1 | +| 1 | 2.1 | 2 | 1 | +| 1 | 4.1 | 4 | 2 | +| 1 | 8.1 | 8 | 8 | ++---+------+----+----+ + +# TODO Uncomment this after tidb bug is adressed +# mysql> use test; set tidb_enforce_mpp=1; select *, first_value(v) over (partition by p order by o range between 2.3E0 preceding and 0 following) as a from test.first_range_d32; +# +---+------+----+----+ +# | p | o | v | a | +# +---+------+----+----+ +# | 0 | 0.1 | 0 | 0 | +# | 1 | 1.0 | 1 | 1 | +# | 1 | 2.1 | 2 | 1 | +# | 1 | 4.1 | 4 | 2 | +# | 1 | 8.1 | 8 | 8 | +# | 2 | 0.0 | 0 | 0 | +# | 2 | 3.1 | 3 | 3 | +# | 2 | 10.0 | 10 | 10 | +# | 2 | 13.1 | 13 | 13 | +# | 2 | 15.1 | 15 | 13 | +# | 3 | 1.1 | 1 | 1 | +# | 3 | 2.9 | 3 | 1 | +# | 3 | 5.1 | 5 | 3 | +# | 3 | 9.1 | 9 | 9 | +# | 3 | 15.0 | 15 | 15 | +# | 3 | 20.1 | 20 | 20 | +# | 3 | 31.1 | 31 | 31 | +# +---+------+----+----+ + +mysql> use test; set tidb_enforce_mpp=1; select *, first_value(v) over (partition by p order by o range between 2.3 preceding and 0 following) as a from test.first_range_d32; ++---+------+----+----+ +| p | o | v | a | ++---+------+----+----+ +| 3 | 1.1 | 1 | 1 | +| 3 | 2.9 | 3 | 1 | +| 3 | 5.1 | 5 | 3 | +| 3 | 9.1 | 9 | 9 | +| 3 | 15.0 | 15 | 15 | +| 3 | 20.1 | 20 | 20 | +| 3 | 31.1 | 31 | 31 | +| 2 | 0.0 | 0 | 0 | +| 2 | 3.1 | 3 | 3 | +| 2 | 10.0 | 10 | 10 | +| 2 | 13.1 | 13 | 13 | +| 2 | 15.1 | 15 | 13 | +| 1 | 1.0 | 1 | 1 | +| 1 | 2.1 | 2 | 1 | +| 1 | 4.1 | 4 | 2 | +| 1 | 8.1 | 8 | 8 | +| 0 | 0.1 | 0 | 0 | ++---+------+----+----+ + +mysql> use test; set tidb_enforce_mpp=1; select *, first_value(v) over (partition by p order by o range between 2.3 preceding and 0 following) as a from test.first_range_d64; ++---+------+----+----+ +| p | o | v | a | ++---+------+----+----+ +| 3 | 1.1 | 1 | 1 | +| 3 | 2.9 | 3 | 1 | +| 3 | 5.1 | 5 | 3 | +| 3 | 9.1 | 9 | 9 | +| 3 | 15.0 | 15 | 15 | +| 3 | 20.1 | 20 | 20 | +| 3 | 31.1 | 31 | 31 | +| 2 | 0.0 | 0 | 0 | +| 2 | 3.1 | 3 | 3 | +| 2 | 10.0 | 10 | 10 | +| 2 | 13.1 | 13 | 13 | +| 2 | 15.1 | 15 | 13 | +| 1 | 1.0 | 1 | 1 | +| 1 | 2.1 | 2 | 1 | +| 1 | 4.1 | 4 | 2 | +| 1 | 8.1 | 8 | 8 | +| 0 | 0.1 | 0 | 0 | ++---+------+----+----+ + +mysql> use test; set tidb_enforce_mpp=1; select *, first_value(v) over (partition by p order by o range between 2.3 preceding and 0 following) as a from test.first_range_d128; ++---+------+----+----+ +| p | o | v | a | ++---+------+----+----+ +| 3 | 1.1 | 1 | 1 | +| 3 | 2.9 | 3 | 1 | +| 3 | 5.1 | 5 | 3 | +| 3 | 9.1 | 9 | 9 | +| 3 | 15.0 | 15 | 15 | +| 3 | 20.1 | 20 | 20 | +| 3 | 31.1 | 31 | 31 | +| 2 | 0.0 | 0 | 0 | +| 2 | 3.1 | 3 | 3 | +| 2 | 10.0 | 10 | 10 | +| 2 | 13.1 | 13 | 13 | +| 2 | 15.1 | 15 | 13 | +| 1 | 1.0 | 1 | 1 | +| 1 | 2.1 | 2 | 1 | +| 1 | 4.1 | 4 | 2 | +| 1 | 8.1 | 8 | 8 | +| 0 | 0.1 | 0 | 0 | ++---+------+----+----+ + +mysql> use test; set tidb_enforce_mpp=1; select *, first_value(v) over (partition by p order by o range between 2.3 preceding and 0 following) as a from test.first_range_d256; ++---+------+----+----+ +| p | o | v | a | ++---+------+----+----+ +| 3 | 1.1 | 1 | 1 | +| 3 | 2.9 | 3 | 1 | +| 3 | 5.1 | 5 | 3 | +| 3 | 9.1 | 9 | 9 | +| 3 | 15.0 | 15 | 15 | +| 3 | 20.1 | 20 | 20 | +| 3 | 31.1 | 31 | 31 | +| 2 | 0.0 | 0 | 0 | +| 2 | 3.1 | 3 | 3 | +| 2 | 10.0 | 10 | 10 | +| 2 | 13.1 | 13 | 13 | +| 2 | 15.1 | 15 | 13 | +| 1 | 1.0 | 1 | 1 | +| 1 | 2.1 | 2 | 1 | +| 1 | 4.1 | 4 | 2 | +| 1 | 8.1 | 8 | 8 | +| 0 | 0.1 | 0 | 0 | ++---+------+----+----+ + +# last_value tests +mysql> drop table if exists test.last_range; +mysql> create table test.last_range(p int not null,o int not null,v int not null); +mysql> insert into test.last_range (p, o, v) values (0, 0, 0), (1, 1, 1), (1, 2, 2), (1, 4, 4), (1, 8, 8), (2, 0, 0), (2, 3, 3), (2, 10, 10), (2, 13, 13), (2, 15, 15), (3, 1, 1), (3, 3, 3), (3, 5, 5), (3, 9, 9), (3, 15, 15), (3, 20, 20), (3, 31, 31); +mysql> alter table test.last_range set tiflash replica 1; + +mysql> drop table if exists test.last_range_f; +mysql> create table test.last_range_f(p int not null, o double not null, v int not null); +mysql> insert into test.last_range_f (p, o, v) values (0, 0.1, 0), (1, 1.2, 1), (1, 2.1, 2), (1, 4.1, 4), (1, 8.1, 8), (2, 0.0, 0), (2, 3.1, 3), (2, 10.0, 10), (2, 13.1, 13), (2, 15.1, 15), (3, 1.1, 1), (3, 2.9, 3), (3, 5.1, 5), (3, 9.1, 9), (3, 15.0, 15), (3, 20.1, 20), (3, 31.1, 31); +mysql> alter table test.last_range_f set tiflash replica 1; + +mysql> drop table if exists test.last_range_d32; +mysql> create table test.last_range_d32(p int not null, o decimal(7,1) not null, v int not null); +mysql> insert into test.last_range_d32 (p, o, v) values (0, 0.1, 0), (1, 1.2, 1), (1, 2.1, 2), (1, 4.1, 4), (1, 8.1, 8), (2, 0.0, 0), (2, 3.1, 3), (2, 10.0, 10), (2, 13.1, 13), (2, 15.1, 15), (3, 1.1, 1), (3, 2.9, 3), (3, 5.1, 5), (3, 9.1, 9), (3, 15.0, 15), (3, 20.1, 20), (3, 31.1, 31); +mysql> alter table test.last_range_d32 set tiflash replica 1; + +mysql> drop table if exists test.last_range_d64; +mysql> create table test.last_range_d64(p int not null, o decimal(17,1) not null, v int not null); +mysql> insert into test.last_range_d64 (p, o, v) values (0, 0.1, 0), (1, 1.2, 1), (1, 2.1, 2), (1, 4.1, 4), (1, 8.1, 8), (2, 0.0, 0), (2, 3.1, 3), (2, 10.0, 10), (2, 13.1, 13), (2, 15.1, 15), (3, 1.1, 1), (3, 2.9, 3), (3, 5.1, 5), (3, 9.1, 9), (3, 15.0, 15), (3, 20.1, 20), (3, 31.1, 31); +mysql> alter table test.last_range_d64 set tiflash replica 1; + +mysql> drop table if exists test.last_range_d128; +mysql> create table test.last_range_d128(p int not null, o decimal(37,1) not null, v int not null); +mysql> insert into test.last_range_d128 (p, o, v) values (0, 0.1, 0), (1, 1.2, 1), (1, 2.1, 2), (1, 4.1, 4), (1, 8.1, 8), (2, 0.0, 0), (2, 3.1, 3), (2, 10.0, 10), (2, 13.1, 13), (2, 15.1, 15), (3, 1.1, 1), (3, 2.9, 3), (3, 5.1, 5), (3, 9.1, 9), (3, 15.0, 15), (3, 20.1, 20), (3, 31.1, 31); +mysql> alter table test.last_range_d128 set tiflash replica 1; + +mysql> drop table if exists test.last_range_d256; +mysql> create table test.last_range_d256(p int not null, o decimal(64,1) not null, v int not null); +mysql> insert into test.last_range_d256 (p, o, v) values (0, 0.1, 0), (1, 1.2, 1), (1, 2.1, 2), (1, 4.1, 4), (1, 8.1, 8), (2, 0.0, 0), (2, 3.1, 3), (2, 10.0, 10), (2, 13.1, 13), (2, 15.1, 15), (3, 1.1, 1), (3, 2.9, 3), (3, 5.1, 5), (3, 9.1, 9), (3, 15.0, 15), (3, 20.1, 20), (3, 31.1, 31); +mysql> alter table test.last_range_d256 set tiflash replica 1; + +func> wait_table test last_range +func> wait_table test last_range_f +func> wait_table test last_range_d32 +func> wait_table test last_range_d64 +func> wait_table test last_range_d128 +func> wait_table test last_range_d256 + +mysql> use test; set tidb_enforce_mpp=1; select *, last_value(v) over (partition by p order by o range between 0 preceding and 3 following) as a from test.last_range; ++---+----+----+----+ +| p | o | v | a | ++---+----+----+----+ +| 1 | 1 | 1 | 4 | +| 1 | 2 | 2 | 4 | +| 1 | 4 | 4 | 4 | +| 1 | 8 | 8 | 8 | +| 3 | 1 | 1 | 3 | +| 3 | 3 | 3 | 5 | +| 3 | 5 | 5 | 5 | +| 3 | 9 | 9 | 9 | +| 3 | 15 | 15 | 15 | +| 3 | 20 | 20 | 20 | +| 3 | 31 | 31 | 31 | +| 2 | 0 | 0 | 3 | +| 2 | 3 | 3 | 3 | +| 2 | 10 | 10 | 13 | +| 2 | 13 | 13 | 15 | +| 2 | 15 | 15 | 15 | +| 0 | 0 | 0 | 0 | ++---+----+----+----+ + +# TODO Uncomment this after tidb bug is adressed +# mysql> use test; set tidb_enforce_mpp=1; select *, last_value(v) over (partition by p order by o range between 0 preceding and 2.9E0 following) as a from test.last_range; +# +---+----+----+----+ +# | p | o | v | a | +# +---+----+----+----+ +# | 1 | 1 | 1 | 2 | +# | 1 | 2 | 2 | 4 | +# | 1 | 4 | 4 | 4 | +# | 1 | 8 | 8 | 8 | +# | 3 | 1 | 1 | 3 | +# | 3 | 3 | 3 | 5 | +# | 3 | 5 | 5 | 5 | +# | 3 | 9 | 9 | 9 | +# | 3 | 15 | 15 | 15 | +# | 3 | 20 | 20 | 20 | +# | 3 | 31 | 31 | 31 | +# | 0 | 0 | 0 | 0 | +# | 2 | 0 | 0 | 0 | +# | 2 | 3 | 3 | 3 | +# | 2 | 10 | 10 | 10 | +# | 2 | 13 | 13 | 15 | +# | 2 | 15 | 15 | 15 | +# +---+----+----+----+ + +# TODO Uncomment this after tidb bug is adressed +# mysql> use test; set tidb_enforce_mpp=1; select *, last_value(v) over (partition by p order by o range between 0 preceding and 2.9 following) as a from test.last_range; +# +---+----+----+----+ +# | p | o | v | a | +# +---+----+----+----+ +# | 2 | 0 | 0 | 0 | +# | 2 | 3 | 3 | 3 | +# | 2 | 10 | 10 | 10 | +# | 2 | 13 | 13 | 15 | +# | 2 | 15 | 15 | 15 | +# | 3 | 1 | 1 | 3 | +# | 3 | 3 | 3 | 5 | +# | 3 | 5 | 5 | 5 | +# | 3 | 9 | 9 | 9 | +# | 3 | 15 | 15 | 15 | +# | 3 | 20 | 20 | 20 | +# | 3 | 31 | 31 | 31 | +# | 1 | 1 | 1 | 2 | +# | 1 | 2 | 2 | 4 | +# | 1 | 4 | 4 | 4 | +# | 1 | 8 | 8 | 8 | +# | 0 | 0 | 0 | 0 | +# +---+----+----+----+ + +mysql> use test; set tidb_enforce_mpp=1; select *, last_value(v) over (partition by p order by o range between 0 preceding and 3 following) as a from test.last_range_f; ++---+------+----+----+ +| p | o | v | a | ++---+------+----+----+ +| 1 | 1.2 | 1 | 4 | +| 1 | 2.1 | 2 | 4 | +| 1 | 4.1 | 4 | 4 | +| 1 | 8.1 | 8 | 8 | +| 3 | 1.1 | 1 | 3 | +| 3 | 2.9 | 3 | 5 | +| 3 | 5.1 | 5 | 5 | +| 3 | 9.1 | 9 | 9 | +| 3 | 15 | 15 | 15 | +| 3 | 20.1 | 20 | 20 | +| 3 | 31.1 | 31 | 31 | +| 0 | 0.1 | 0 | 0 | +| 2 | 0 | 0 | 0 | +| 2 | 3.1 | 3 | 3 | +| 2 | 10 | 10 | 10 | +| 2 | 13.1 | 13 | 15 | +| 2 | 15.1 | 15 | 15 | ++---+------+----+----+ + +mysql> use test; set tidb_enforce_mpp=1; select *, last_value(v) over (partition by p order by o range between 0 preceding and 2.3E0 following) as a from test.last_range_f; ++---+------+----+----+ +| p | o | v | a | ++---+------+----+----+ +| 0 | 0.1 | 0 | 0 | +| 1 | 1.2 | 1 | 2 | +| 1 | 2.1 | 2 | 4 | +| 1 | 4.1 | 4 | 4 | +| 1 | 8.1 | 8 | 8 | +| 3 | 1.1 | 1 | 3 | +| 3 | 2.9 | 3 | 5 | +| 3 | 5.1 | 5 | 5 | +| 3 | 9.1 | 9 | 9 | +| 3 | 15 | 15 | 15 | +| 3 | 20.1 | 20 | 20 | +| 3 | 31.1 | 31 | 31 | +| 2 | 0 | 0 | 0 | +| 2 | 3.1 | 3 | 3 | +| 2 | 10 | 10 | 10 | +| 2 | 13.1 | 13 | 15 | +| 2 | 15.1 | 15 | 15 | ++---+------+----+----+ + +mysql> use test; set tidb_enforce_mpp=1; select *, last_value(v) over (partition by p order by o range between 0 preceding and 2.3 following) as a from test.last_range_f; ++---+------+----+----+ +| p | o | v | a | ++---+------+----+----+ +| 1 | 1.2 | 1 | 2 | +| 1 | 2.1 | 2 | 4 | +| 1 | 4.1 | 4 | 4 | +| 1 | 8.1 | 8 | 8 | +| 3 | 1.1 | 1 | 3 | +| 3 | 2.9 | 3 | 5 | +| 3 | 5.1 | 5 | 5 | +| 3 | 9.1 | 9 | 9 | +| 3 | 15 | 15 | 15 | +| 3 | 20.1 | 20 | 20 | +| 3 | 31.1 | 31 | 31 | +| 0 | 0.1 | 0 | 0 | +| 2 | 0 | 0 | 0 | +| 2 | 3.1 | 3 | 3 | +| 2 | 10 | 10 | 10 | +| 2 | 13.1 | 13 | 15 | +| 2 | 15.1 | 15 | 15 | ++---+------+----+----+ + +mysql> use test; set tidb_enforce_mpp=1; select *, last_value(v) over (partition by p order by o range between 0 preceding and 3 following) as a from test.last_range_d32; ++---+------+----+----+ +| p | o | v | a | ++---+------+----+----+ +| 1 | 1.2 | 1 | 4 | +| 1 | 2.1 | 2 | 4 | +| 1 | 4.1 | 4 | 4 | +| 1 | 8.1 | 8 | 8 | +| 2 | 0.0 | 0 | 0 | +| 2 | 3.1 | 3 | 3 | +| 2 | 10.0 | 10 | 10 | +| 2 | 13.1 | 13 | 15 | +| 2 | 15.1 | 15 | 15 | +| 3 | 1.1 | 1 | 3 | +| 3 | 2.9 | 3 | 5 | +| 3 | 5.1 | 5 | 5 | +| 3 | 9.1 | 9 | 9 | +| 3 | 15.0 | 15 | 15 | +| 3 | 20.1 | 20 | 20 | +| 3 | 31.1 | 31 | 31 | +| 0 | 0.1 | 0 | 0 | ++---+------+----+----+ + +# TODO Uncomment this after tidb bug is adressed +# mysql> use test; set tidb_enforce_mpp=1; select *, last_value(v) over (partition by p order by o range between 0 preceding and 2.3E0 following) as a from test.last_range_d32; +# +---+------+----+----+ +# | p | o | v | a | +# +---+------+----+----+ +# | 1 | 1.2 | 1 | 2 | +# | 1 | 2.1 | 2 | 4 | +# | 1 | 4.1 | 4 | 4 | +# | 1 | 8.1 | 8 | 8 | +# | 0 | 0.1 | 0 | 0 | +# | 2 | 0.0 | 0 | 0 | +# | 2 | 3.1 | 3 | 3 | +# | 2 | 10.0 | 10 | 10 | +# | 2 | 13.1 | 13 | 15 | +# | 2 | 15.1 | 15 | 15 | +# | 3 | 1.1 | 1 | 3 | +# | 3 | 2.9 | 3 | 5 | +# | 3 | 5.1 | 5 | 5 | +# | 3 | 9.1 | 9 | 9 | +# | 3 | 15.0 | 15 | 15 | +# | 3 | 20.1 | 20 | 20 | +# | 3 | 31.1 | 31 | 31 | +# +---+------+----+----+ + +mysql> use test; set tidb_enforce_mpp=1; select *, last_value(v) over (partition by p order by o range between 0 preceding and 2.3 following) as a from test.last_range_d32; ++---+------+----+----+ +| p | o | v | a | ++---+------+----+----+ +| 1 | 1.2 | 1 | 2 | +| 1 | 2.1 | 2 | 4 | +| 1 | 4.1 | 4 | 4 | +| 1 | 8.1 | 8 | 8 | +| 0 | 0.1 | 0 | 0 | +| 3 | 1.1 | 1 | 3 | +| 3 | 2.9 | 3 | 5 | +| 3 | 5.1 | 5 | 5 | +| 3 | 9.1 | 9 | 9 | +| 3 | 15.0 | 15 | 15 | +| 3 | 20.1 | 20 | 20 | +| 3 | 31.1 | 31 | 31 | +| 2 | 0.0 | 0 | 0 | +| 2 | 3.1 | 3 | 3 | +| 2 | 10.0 | 10 | 10 | +| 2 | 13.1 | 13 | 15 | +| 2 | 15.1 | 15 | 15 | ++---+------+----+----+ + +mysql> use test; set tidb_enforce_mpp=1; select *, last_value(v) over (partition by p order by o range between 0 preceding and 2.3 following) as a from test.last_range_d64; ++---+------+----+----+ +| p | o | v | a | ++---+------+----+----+ +| 1 | 1.2 | 1 | 2 | +| 1 | 2.1 | 2 | 4 | +| 1 | 4.1 | 4 | 4 | +| 1 | 8.1 | 8 | 8 | +| 0 | 0.1 | 0 | 0 | +| 3 | 1.1 | 1 | 3 | +| 3 | 2.9 | 3 | 5 | +| 3 | 5.1 | 5 | 5 | +| 3 | 9.1 | 9 | 9 | +| 3 | 15.0 | 15 | 15 | +| 3 | 20.1 | 20 | 20 | +| 3 | 31.1 | 31 | 31 | +| 2 | 0.0 | 0 | 0 | +| 2 | 3.1 | 3 | 3 | +| 2 | 10.0 | 10 | 10 | +| 2 | 13.1 | 13 | 15 | +| 2 | 15.1 | 15 | 15 | ++---+------+----+----+ + +mysql> use test; set tidb_enforce_mpp=1; select *, last_value(v) over (partition by p order by o range between 0 preceding and 2.3 following) as a from test.last_range_d128; ++---+------+----+----+ +| p | o | v | a | ++---+------+----+----+ +| 1 | 1.2 | 1 | 2 | +| 1 | 2.1 | 2 | 4 | +| 1 | 4.1 | 4 | 4 | +| 1 | 8.1 | 8 | 8 | +| 0 | 0.1 | 0 | 0 | +| 3 | 1.1 | 1 | 3 | +| 3 | 2.9 | 3 | 5 | +| 3 | 5.1 | 5 | 5 | +| 3 | 9.1 | 9 | 9 | +| 3 | 15.0 | 15 | 15 | +| 3 | 20.1 | 20 | 20 | +| 3 | 31.1 | 31 | 31 | +| 2 | 0.0 | 0 | 0 | +| 2 | 3.1 | 3 | 3 | +| 2 | 10.0 | 10 | 10 | +| 2 | 13.1 | 13 | 15 | +| 2 | 15.1 | 15 | 15 | ++---+------+----+----+ + +mysql> use test; set tidb_enforce_mpp=1; select *, last_value(v) over (partition by p order by o range between 0 preceding and 2.3 following) as a from test.last_range_d256; ++---+------+----+----+ +| p | o | v | a | ++---+------+----+----+ +| 1 | 1.2 | 1 | 2 | +| 1 | 2.1 | 2 | 4 | +| 1 | 4.1 | 4 | 4 | +| 1 | 8.1 | 8 | 8 | +| 0 | 0.1 | 0 | 0 | +| 3 | 1.1 | 1 | 3 | +| 3 | 2.9 | 3 | 5 | +| 3 | 5.1 | 5 | 5 | +| 3 | 9.1 | 9 | 9 | +| 3 | 15.0 | 15 | 15 | +| 3 | 20.1 | 20 | 20 | +| 3 | 31.1 | 31 | 31 | +| 2 | 0.0 | 0 | 0 | +| 2 | 3.1 | 3 | 3 | +| 2 | 10.0 | 10 | 10 | +| 2 | 13.1 | 13 | 15 | +| 2 | 15.1 | 15 | 15 | ++---+------+----+----+ + +mysql> drop table if exists test.test_interval; +mysql> create table test.test_interval(p int not null,o datetime not null,v int not null); +mysql> insert into test.test_interval (p, o, v) values (0, '2023-5-15 10:21:21', 1), (1, '2023-5-14 10:21:21', 2), (1, '2023-5-15 10:21:21', 3), (1, '2023-5-15 10:21:25', 4), (1, '2023-5-15 10:25:21', 5), (2, '2023-4-15 10:21:21', 6), (2, '2023-5-14 10:21:21', 7), (2, '2023-5-15 10:21:25', 8), (2, '2023-5-15 10:25:21', 9), (2, '2023-5-15 11:21:21', 10), (3, '2022-5-15 10:21:21', 11), (3, '2023-5-15 10:21:21', 12), (3, '2024-5-15 10:21:21', 13); +mysql> alter table test.test_interval set tiflash replica 1; + +func> wait_table test test_interval + +mysql> use test; set tidb_enforce_mpp=1; select *, first_value(v) over (partition by p order by o range between interval 2 day preceding and interval 2 day following) as a from test_interval; ++---+---------------------+----+------+ +| p | o | v | a | ++---+---------------------+----+------+ +| 0 | 2023-05-15 10:21:21 | 1 | 1 | +| 1 | 2023-05-14 10:21:21 | 2 | 2 | +| 1 | 2023-05-15 10:21:21 | 3 | 2 | +| 1 | 2023-05-15 10:21:25 | 4 | 2 | +| 1 | 2023-05-15 10:25:21 | 5 | 2 | +| 2 | 2023-04-15 10:21:21 | 6 | 6 | +| 2 | 2023-05-14 10:21:21 | 7 | 7 | +| 2 | 2023-05-15 10:21:25 | 8 | 7 | +| 2 | 2023-05-15 10:25:21 | 9 | 7 | +| 2 | 2023-05-15 11:21:21 | 10 | 7 | +| 3 | 2022-05-15 10:21:21 | 11 | 11 | +| 3 | 2023-05-15 10:21:21 | 12 | 12 | +| 3 | 2024-05-15 10:21:21 | 13 | 13 | ++---+---------------------+----+------+ + +mysql> use test; set tidb_enforce_mpp=1; select *, first_value(v) over (partition by p order by o range between interval 1 year preceding and interval 1 year following) as a from test_interval; ++---+---------------------+----+------+ +| p | o | v | a | ++---+---------------------+----+------+ +| 0 | 2023-05-15 10:21:21 | 1 | 1 | +| 1 | 2023-05-14 10:21:21 | 2 | 2 | +| 1 | 2023-05-15 10:21:21 | 3 | 2 | +| 1 | 2023-05-15 10:21:25 | 4 | 2 | +| 1 | 2023-05-15 10:25:21 | 5 | 2 | +| 2 | 2023-04-15 10:21:21 | 6 | 6 | +| 2 | 2023-05-14 10:21:21 | 7 | 6 | +| 2 | 2023-05-15 10:21:25 | 8 | 6 | +| 2 | 2023-05-15 10:25:21 | 9 | 6 | +| 2 | 2023-05-15 11:21:21 | 10 | 6 | +| 3 | 2022-05-15 10:21:21 | 11 | 11 | +| 3 | 2023-05-15 10:21:21 | 12 | 11 | +| 3 | 2024-05-15 10:21:21 | 13 | 12 | ++---+---------------------+----+------+ + +mysql> drop table if exists test.null_order; +mysql> create table test.null_order (p int not null,o int,v int not null); +mysql> insert into test.null_order (p, o, v) values (0, 0, 1), (1, null, 2), (1, 1, 3), (1, 2, 4), (2, null, 5), (2, 5, 6), (2, 8, 7), (2, 10, 8), (2, 12, 9), (2, 15, 10); +mysql> alter table test.null_order set tiflash replica 1; + +func> wait_table test null_order + +mysql> use test; set tidb_enforce_mpp=1; select *, first_value(v) over (partition by p order by o asc range between 1 following and 2 following) as a from null_order; ++---+------+------+------+ +| p | o | v | a | ++---+------+------+------+ +| 0 | 0 | 1 | NULL | +| 1 | NULL | 2 | 2 | +| 1 | 1 | 3 | 4 | +| 1 | 2 | 4 | NULL | +| 2 | NULL | 5 | 5 | +| 2 | 5 | 6 | NULL | +| 2 | 8 | 7 | 8 | +| 2 | 10 | 8 | 9 | +| 2 | 12 | 9 | NULL | +| 2 | 15 | 10 | NULL | ++---+------+------+------+ + +mysql> use test; set tidb_enforce_mpp=1; select *, first_value(v) over (partition by p order by o ASC RANGE BETWEEN 2 FOLLOWING AND UNBOUNDED FOLLOWING) as a from null_order; ++---+------+------+------+ +| p | o | v | a | ++---+------+------+------+ +| 0 | 0 | 1 | NULL | +| 1 | NULL | 2 | 2 | +| 1 | 1 | 3 | NULL | +| 1 | 2 | 4 | NULL | +| 2 | NULL | 5 | 5 | +| 2 | 5 | 6 | 7 | +| 2 | 8 | 7 | 8 | +| 2 | 10 | 8 | 9 | +| 2 | 12 | 9 | 10 | +| 2 | 15 | 10 | NULL | ++---+------+------+------+ + +mysql> use test; set tidb_enforce_mpp=1; select *, first_value(v) over (partition by p order by o DESC RANGE BETWEEN 2 FOLLOWING AND UNBOUNDED FOLLOWING) as a from null_order; ++---+------+------+------+ +| p | o | v | a | ++---+------+------+------+ +| 0 | 0 | 1 | NULL | +| 1 | 2 | 4 | 2 | +| 1 | 1 | 3 | 2 | +| 1 | NULL | 2 | 2 | +| 2 | 15 | 10 | 9 | +| 2 | 12 | 9 | 8 | +| 2 | 10 | 8 | 7 | +| 2 | 8 | 7 | 6 | +| 2 | 5 | 6 | 5 | +| 2 | NULL | 5 | 5 | ++---+------+------+------+ + +mysql> use test; set tidb_enforce_mpp=1; select *, first_value(v) over (partition by p order by o ASC RANGE BETWEEN 11 PRECEDING AND UNBOUNDED FOLLOWING) as a from null_order; ++---+------+------+------+ +| p | o | v | a | ++---+------+------+------+ +| 0 | 0 | 1 | 1 | +| 1 | NULL | 2 | 2 | +| 1 | 1 | 3 | 3 | +| 1 | 2 | 4 | 3 | +| 2 | NULL | 5 | 5 | +| 2 | 5 | 6 | 6 | +| 2 | 8 | 7 | 6 | +| 2 | 10 | 8 | 6 | +| 2 | 12 | 9 | 6 | +| 2 | 15 | 10 | 6 | ++---+------+------+------+ + +mysql> use test; set tidb_enforce_mpp=1; select *, first_value(v) over (partition by p order by o ASC RANGE BETWEEN 1 PRECEDING AND 11 FOLLOWING) as a from null_order; ++---+------+------+------+ +| p | o | v | a | ++---+------+------+------+ +| 0 | 0 | 1 | 1 | +| 1 | NULL | 2 | 2 | +| 1 | 1 | 3 | 3 | +| 1 | 2 | 4 | 3 | +| 2 | NULL | 5 | 5 | +| 2 | 5 | 6 | 6 | +| 2 | 8 | 7 | 7 | +| 2 | 10 | 8 | 8 | +| 2 | 12 | 9 | 9 | +| 2 | 15 | 10 | 10 | ++---+------+------+------+ + +mysql> use test; set tidb_enforce_mpp=1; select *, first_value(v) over (partition by p order by o ASC RANGE BETWEEN 2 PRECEDING AND 1 PRECEDING) as a from null_order; ++---+------+------+------+ +| p | o | v | a | ++---+------+------+------+ +| 0 | 0 | 1 | NULL | +| 1 | NULL | 2 | 2 | +| 1 | 1 | 3 | NULL | +| 1 | 2 | 4 | 3 | +| 2 | NULL | 5 | 5 | +| 2 | 5 | 6 | NULL | +| 2 | 8 | 7 | NULL | +| 2 | 10 | 8 | 7 | +| 2 | 12 | 9 | 8 | +| 2 | 15 | 10 | NULL | ++---+------+------+------+ + +mysql> use test; set tidb_enforce_mpp=1; select *, first_value(v) over (partition by p order by o ASC RANGE BETWEEN UNBOUNDED PRECEDING AND 2 FOLLOWING) as a from null_order; ++---+------+------+------+ +| p | o | v | a | ++---+------+------+------+ +| 0 | 0 | 1 | 1 | +| 1 | NULL | 2 | 2 | +| 1 | 1 | 3 | 2 | +| 1 | 2 | 4 | 2 | +| 2 | NULL | 5 | 5 | +| 2 | 5 | 6 | 5 | +| 2 | 8 | 7 | 5 | +| 2 | 10 | 8 | 5 | +| 2 | 12 | 9 | 5 | +| 2 | 15 | 10 | 5 | ++---+------+------+------+ + +mysql> use test; set tidb_enforce_mpp=1; select *, first_value(v) over (partition by p ORDER BY o DESC RANGE BETWEEN 2 FOLLOWING AND 2 FOLLOWING) as a from null_order; ++---+------+------+------+ +| p | o | v | a | ++---+------+------+------+ +| 2 | 15 | 10 | NULL | +| 2 | 12 | 9 | 8 | +| 2 | 10 | 8 | 7 | +| 2 | 8 | 7 | NULL | +| 2 | 5 | 6 | NULL | +| 2 | NULL | 5 | 5 | +| 1 | 2 | 4 | NULL | +| 1 | 1 | 3 | NULL | +| 1 | NULL | 2 | 2 | +| 0 | 0 | 1 | NULL | ++---+------+------+------+ + +mysql> use test; set tidb_enforce_mpp=1; select *, last_value(v) over (partition by p order by o asc range between 1 following and 2 following) as a from null_order; ++---+------+------+------+ +| p | o | v | a | ++---+------+------+------+ +| 0 | 0 | 1 | NULL | +| 1 | NULL | 2 | 2 | +| 1 | 1 | 3 | 4 | +| 1 | 2 | 4 | NULL | +| 2 | NULL | 5 | 5 | +| 2 | 5 | 6 | NULL | +| 2 | 8 | 7 | 8 | +| 2 | 10 | 8 | 9 | +| 2 | 12 | 9 | NULL | +| 2 | 15 | 10 | NULL | ++---+------+------+------+ + +mysql> use test; set tidb_enforce_mpp=1; select *, last_value(v) over (partition by p order by o ASC RANGE BETWEEN 2 FOLLOWING AND UNBOUNDED FOLLOWING) as a from null_order; ++---+------+------+------+ +| p | o | v | a | ++---+------+------+------+ +| 0 | 0 | 1 | NULL | +| 1 | NULL | 2 | 4 | +| 1 | 1 | 3 | NULL | +| 1 | 2 | 4 | NULL | +| 2 | NULL | 5 | 10 | +| 2 | 5 | 6 | 10 | +| 2 | 8 | 7 | 10 | +| 2 | 10 | 8 | 10 | +| 2 | 12 | 9 | 10 | +| 2 | 15 | 10 | NULL | ++---+------+------+------+ + +mysql> use test; set tidb_enforce_mpp=1; select *, last_value(v) over (partition by p order by o DESC RANGE BETWEEN 2 FOLLOWING AND UNBOUNDED FOLLOWING) as a from null_order; ++---+------+------+------+ +| p | o | v | a | ++---+------+------+------+ +| 0 | 0 | 1 | NULL | +| 1 | 2 | 4 | 2 | +| 1 | 1 | 3 | 2 | +| 1 | NULL | 2 | 2 | +| 2 | 15 | 10 | 5 | +| 2 | 12 | 9 | 5 | +| 2 | 10 | 8 | 5 | +| 2 | 8 | 7 | 5 | +| 2 | 5 | 6 | 5 | +| 2 | NULL | 5 | 5 | ++---+------+------+------+ + +mysql> use test; set tidb_enforce_mpp=1; select *, last_value(v) over (partition by p order by o ASC RANGE BETWEEN 11 PRECEDING AND UNBOUNDED FOLLOWING) as a from null_order; ++---+------+------+------+ +| p | o | v | a | ++---+------+------+------+ +| 0 | 0 | 1 | 1 | +| 1 | NULL | 2 | 4 | +| 1 | 1 | 3 | 4 | +| 1 | 2 | 4 | 4 | +| 2 | NULL | 5 | 10 | +| 2 | 5 | 6 | 10 | +| 2 | 8 | 7 | 10 | +| 2 | 10 | 8 | 10 | +| 2 | 12 | 9 | 10 | +| 2 | 15 | 10 | 10 | ++---+------+------+------+ + +mysql> use test; set tidb_enforce_mpp=1; select *, last_value(v) over (partition by p order by o ASC RANGE BETWEEN 1 PRECEDING AND 11 FOLLOWING) as a from null_order; ++---+------+------+------+ +| p | o | v | a | ++---+------+------+------+ +| 0 | 0 | 1 | 1 | +| 1 | NULL | 2 | 2 | +| 1 | 1 | 3 | 4 | +| 1 | 2 | 4 | 4 | +| 2 | NULL | 5 | 5 | +| 2 | 5 | 6 | 10 | +| 2 | 8 | 7 | 10 | +| 2 | 10 | 8 | 10 | +| 2 | 12 | 9 | 10 | +| 2 | 15 | 10 | 10 | ++---+------+------+------+ + +mysql> use test; set tidb_enforce_mpp=1; select *, last_value(v) over (partition by p order by o ASC RANGE BETWEEN 2 PRECEDING AND 1 PRECEDING) as a from null_order; ++---+------+------+------+ +| p | o | v | a | ++---+------+------+------+ +| 0 | 0 | 1 | NULL | +| 1 | NULL | 2 | 2 | +| 1 | 1 | 3 | NULL | +| 1 | 2 | 4 | 3 | +| 2 | NULL | 5 | 5 | +| 2 | 5 | 6 | NULL | +| 2 | 8 | 7 | NULL | +| 2 | 10 | 8 | 7 | +| 2 | 12 | 9 | 8 | +| 2 | 15 | 10 | NULL | ++---+------+------+------+ + +mysql> use test; set tidb_enforce_mpp=1; select *, last_value(v) over (partition by p order by o ASC RANGE BETWEEN UNBOUNDED PRECEDING AND 2 FOLLOWING) as a from null_order; ++---+------+------+------+ +| p | o | v | a | ++---+------+------+------+ +| 0 | 0 | 1 | 1 | +| 1 | NULL | 2 | 2 | +| 1 | 1 | 3 | 4 | +| 1 | 2 | 4 | 4 | +| 2 | NULL | 5 | 5 | +| 2 | 5 | 6 | 6 | +| 2 | 8 | 7 | 8 | +| 2 | 10 | 8 | 9 | +| 2 | 12 | 9 | 9 | +| 2 | 15 | 10 | 10 | ++---+------+------+------+ + +mysql> use test; set tidb_enforce_mpp=1; select *, last_value(v) over (partition by p ORDER BY o DESC RANGE BETWEEN 2 FOLLOWING AND 2 FOLLOWING) as a from null_order; ++---+------+------+------+ +| p | o | v | a | ++---+------+------+------+ +| 1 | 2 | 4 | NULL | +| 1 | 1 | 3 | NULL | +| 1 | NULL | 2 | 2 | +| 0 | 0 | 1 | NULL | +| 2 | 15 | 10 | NULL | +| 2 | 12 | 9 | 8 | +| 2 | 10 | 8 | 7 | +| 2 | 8 | 7 | NULL | +| 2 | 5 | 6 | NULL | +| 2 | NULL | 5 | 5 | ++---+------+------+------+