From a4a82e655365a295a0832525cd4be44bdb6aa492 Mon Sep 17 00:00:00 2001 From: stephen <91597003+stephen-shelby@users.noreply.github.com> Date: Thu, 9 Jan 2025 10:15:45 +0800 Subject: [PATCH] [Enhancement] adjust translate function syntax structure to prevent parser performance rollback (#54830) Signed-off-by: stephen (cherry picked from commit c1bc8d31d114dba7232a683d4cb3581ace2132d7) # Conflicts: # fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 --- .../java/com/starrocks/sql/parser/AstBuilder.java | 12 +++++++++++- .../main/java/com/starrocks/sql/parser/StarRocks.g4 | 5 +++++ .../java/com/starrocks/sql/parser/StarRocksLex.g4 | 2 +- .../java/com/starrocks/sql/parser/ParserTest.java | 12 ++++++++++++ test/sql/test_agg/R/test_distinct_agg | 11 ++++------- test/sql/test_agg/T/test_distinct_agg | 6 +++--- 6 files changed, 36 insertions(+), 12 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java b/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java index 82340af098c6b3..fa05ce03ed5ae2 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java @@ -6271,7 +6271,6 @@ private static void addArgumentUseTypeInt(Expr value, List exprs) { @Override public ParseNode visitSimpleFunctionCall(StarRocksParser.SimpleFunctionCallContext context) { - String fullFunctionName = getQualifiedName(context.qualifiedName()).toString(); NodePosition pos = createPos(context); @@ -6471,6 +6470,17 @@ public ParseNode visitSimpleFunctionCall(StarRocksParser.SimpleFunctionCallConte return SyntaxSugars.parse(functionCallExpr); } + @Override + public ParseNode visitTranslateFunctionCall(StarRocksParser.TranslateFunctionCallContext context) { + String fullFunctionName = context.TRANSLATE().getText(); + NodePosition pos = createPos(context); + + FunctionName fnName = FunctionName.createFnName(fullFunctionName); + FunctionCallExpr functionCallExpr = new FunctionCallExpr(fnName, + new FunctionParams(false, visit(context.expression(), Expr.class)), pos); + return SyntaxSugars.parse(functionCallExpr); + } + @Override public ParseNode visitAggregationFunctionCall(StarRocksParser.AggregationFunctionCallContext context) { NodePosition pos = createPos(context); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 b/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 index e4b2a97796c2a8..9bd195350fb2cf 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 +++ b/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 @@ -2322,6 +2322,7 @@ functionCall | specialFunctionExpression #specialFunction | aggregationFunction over? #aggregationFunctionCall | windowFunction over #windowFunctionCall + | TRANSLATE '(' (expression (',' expression)*)? ')' #translateFunctionCall | qualifiedName '(' (expression (',' expression)*)? ')' over? #simpleFunctionCall ; @@ -2780,7 +2781,11 @@ nonReserved | SAMPLE | SCHEDULE | SCHEDULER | SECOND | SECURITY | SEPARATOR | SERIALIZABLE |SEMI | SESSION | SETS | SIGNED | SNAPSHOT | SQLBLACKLIST | START | STREAM | SUM | STATUS | STOP | SKIP_HEADER | SWAP | STORAGE| STRING | STRUCT | STATS | SUBMIT | SUSPEND | SYNC | SYSTEM_TIME +<<<<<<< HEAD | TABLES | TABLET | TABLETS | TASK | TEMPORARY | TIMESTAMP | TIMESTAMPADD | TIMESTAMPDIFF | THAN | TIME | TIMES | TRANSACTION | TRACE +======= + | TABLES | TABLET | TABLETS | TAG | TASK | TEMPORARY | TIMESTAMP | TIMESTAMPADD | TIMESTAMPDIFF | THAN | TIME | TIMES | TRANSACTION | TRACE | TRANSLATE +>>>>>>> c1bc8d31d1 ([Enhancement] adjust translate function syntax structure to prevent parser performance rollback (#54830)) | TRIM_SPACE | TRIGGERS | TRUNCATE | TYPE | TYPES | UNBOUNDED | UNCOMMITTED | UNSET | UNINSTALL | USAGE | USER | USERS | UNLOCK diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocksLex.g4 b/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocksLex.g4 index 23c5fb3561d912..7605e8a3152725 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocksLex.g4 +++ b/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocksLex.g4 @@ -429,7 +429,7 @@ TIMESTAMPADD: 'TIMESTAMPADD'; TIMESTAMPDIFF: 'TIMESTAMPDIFF'; TINYINT: 'TINYINT'; TRANSACTION: 'TRANSACTION'; -TRANSLATE: {getCharPositionInLine() == 0}? 'TRANSLATE'; +TRANSLATE: 'TRANSLATE'; TO: 'TO'; TRACE: 'TRACE'; TRIGGERS: 'TRIGGERS'; diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/parser/ParserTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/parser/ParserTest.java index 9c66c4d9f3dbf9..1f0e20bfc6796c 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/parser/ParserTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/parser/ParserTest.java @@ -41,6 +41,7 @@ import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.atn.PredictionMode; import org.junit.Assert; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -587,6 +588,17 @@ private static Stream unexpectedTokenSqls() { return arguments.stream(); } + @Test + public void testTranslateFunction() { + String sql = "select translate('abcabc', 'ab', '12') as test;"; + SessionVariable sessionVariable = new SessionVariable(); + try { + SqlParser.parse(sql, sessionVariable); + } catch (Exception e) { + Assertions.fail("sql should success. errMsg: " + e.getMessage()); + } + } + } diff --git a/test/sql/test_agg/R/test_distinct_agg b/test/sql/test_agg/R/test_distinct_agg index 613f28e39c59d3..7be65a3c20dc38 100644 --- a/test/sql/test_agg/R/test_distinct_agg +++ b/test/sql/test_agg/R/test_distinct_agg @@ -217,10 +217,9 @@ select count(distinct t1.c1, if(t2.c3 is null, 1, 0)), sum(t1.c2 + if(t2.c3 is n -- result: 1 5000 -- !result -# streaming_preaggregation_mode = 'auto' new_planner_agg_stage = '0' +-- streaming_preaggregation_mode = 'auto' new_planner_agg_stage = '0' set streaming_preaggregation_mode = 'auto'; -- result: -E: (1064, "Getting syntax error at line 1, column 2. Detail message: Unexpected input 'streaming_preaggregation_mode', the most similar input is {'ADMIN', 'GRANT', TRANSLATE, 'STOP', 'TRUNCATE', '(', ';'}.") -- !result set new_planner_agg_stage = '0'; -- result: @@ -489,10 +488,9 @@ select array_agg(distinct c2), count(distinct c2), sum(c3) from skew_agg group b [3] 1 5000 [3] 1 5000 -- !result -# streaming_preaggregation_mode = 'force_streaming' new_planner_agg_stage = '2' +-- streaming_preaggregation_mode = 'force_streaming' new_planner_agg_stage = '2' set streaming_preaggregation_mode = 'force_streaming'; -- result: -E: (1064, "Getting syntax error at line 1, column 2. Detail message: Unexpected input 'streaming_preaggregation_mode', the most similar input is {'ADMIN', 'GRANT', TRANSLATE, 'STOP', 'TRUNCATE', '(', ';'}.") -- !result set new_planner_agg_stage = '0'; -- result: @@ -737,10 +735,9 @@ select array_agg(distinct c2), count(distinct c2), sum(c3) from skew_agg group b [3] 1 5000 [3] 1 5000 -- !result -# streaming_preaggregation_mode = 'force_preaggregation' new_planner_agg_stage = '2' +-- streaming_preaggregation_mode = 'force_preaggregation' new_planner_agg_stage = '2' set streaming_preaggregation_mode = 'force_preaggregation'; -- result: -E: (1064, "Getting syntax error at line 1, column 2. Detail message: Unexpected input 'streaming_preaggregation_mode', the most similar input is {'ADMIN', 'GRANT', TRANSLATE, 'STOP', 'TRUNCATE', '(', ';'}.") -- !result set new_planner_agg_stage = '0'; -- result: @@ -1008,4 +1005,4 @@ select sum(c0), count(distinct c2, c3) from skew_agg where c0 = 480816 and c1 = select count(distinct 1, 2), count(distinct 1), count(distinct null), group_concat(distinct 'a', 'b'), count(distinct c2) from skew_agg; -- result: 1 1 0 ab 1 --- !result \ No newline at end of file +-- !result diff --git a/test/sql/test_agg/T/test_distinct_agg b/test/sql/test_agg/T/test_distinct_agg index f88230e24476ca..16d654e52981f4 100644 --- a/test/sql/test_agg/T/test_distinct_agg +++ b/test/sql/test_agg/T/test_distinct_agg @@ -90,7 +90,7 @@ select count(distinct c1, c2), count(distinct c2, c3), sum(c2) from skew_agg lim select count(distinct c1, c2, c3), count(distinct c2, c3), sum(c2) from skew_agg group by c4 limit 500; select count(distinct t1.c1, if(t2.c3 is null, 1, 0)), sum(t1.c2 + if(t2.c3 is null, 1, 0)) from skew_agg t1 left join skew_agg t2 on t1.c4 > t2.c4 group by t1.c4 limit 500; -# streaming_preaggregation_mode = 'auto' new_planner_agg_stage = '0' +-- streaming_preaggregation_mode = 'auto' new_planner_agg_stage = '0' set streaming_preaggregation_mode = 'auto'; set new_planner_agg_stage = '0'; select group_concat(distinct c2, upper(c4) order by abs(c2 + c3)), array_agg(c3 order by 1, c4), ceil(sum(c5)) from (select * from skew_agg order by 1,2,3,4,5,6 limit 10) t; @@ -171,7 +171,7 @@ select group_concat(distinct c2), array_agg(distinct c2), count(distinct c2), su select array_agg(distinct c2), count(distinct c2), sum(c3) from skew_agg group by rollup(c1, c2); -# streaming_preaggregation_mode = 'force_streaming' new_planner_agg_stage = '2' +-- streaming_preaggregation_mode = 'force_streaming' new_planner_agg_stage = '2' set streaming_preaggregation_mode = 'force_streaming'; set new_planner_agg_stage = '0'; select group_concat(distinct c2, upper(c4) order by abs(c2 + c3)), array_agg(c3 order by 1, c4), ceil(sum(c5)) from (select * from skew_agg order by 1,2,3,4,5,6 limit 10) t; @@ -247,7 +247,7 @@ select group_concat(distinct c2), array_agg(distinct c2), count(distinct c2), su select array_agg(distinct c2), count(distinct c2), sum(c3) from skew_agg group by rollup(c1, c2); -# streaming_preaggregation_mode = 'force_preaggregation' new_planner_agg_stage = '2' +-- streaming_preaggregation_mode = 'force_preaggregation' new_planner_agg_stage = '2' set streaming_preaggregation_mode = 'force_preaggregation'; set new_planner_agg_stage = '0'; select group_concat(distinct c2, upper(c4) order by abs(c2 + c3)), array_agg(c3 order by 1, c4), ceil(sum(c5)) from (select * from skew_agg order by 1,2,3,4,5,6 limit 10) t;