From 7aebc3a0f4ec474f3bf5efaae17ae13de6411821 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 29 Jan 2025 21:11:24 +0200 Subject: [PATCH 1/2] fix(databricks-jdbc-driver): Fix extract epoch from timestamp SQL Generation --- packages/cubejs-databricks-jdbc-driver/src/DatabricksQuery.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-databricks-jdbc-driver/src/DatabricksQuery.ts b/packages/cubejs-databricks-jdbc-driver/src/DatabricksQuery.ts index af30172a76e40..afe136753952a 100644 --- a/packages/cubejs-databricks-jdbc-driver/src/DatabricksQuery.ts +++ b/packages/cubejs-databricks-jdbc-driver/src/DatabricksQuery.ts @@ -197,7 +197,7 @@ export class DatabricksQuery extends BaseQuery { templates.functions.GREATEST = 'GREATEST({{ args_concat }})'; templates.functions.TRUNC = 'CASE WHEN ({{ args[0] }}) >= 0 THEN FLOOR({{ args_concat }}) ELSE CEIL({{ args_concat }}) END'; templates.expressions.timestamp_literal = 'from_utc_timestamp(\'{{ value }}\', \'UTC\')'; - templates.expressions.extract = 'EXTRACT({{ date_part }} FROM {{ expr }})'; + templates.expressions.extract = '{% if date_part|lower == "epoch" %}unix_timestamp({{ expr }}){% else %}EXTRACT({{ date_part }} FROM {{ expr }}){% endif %}'; templates.expressions.interval_single_date_part = 'INTERVAL \'{{ num }}\' {{ date_part }}'; templates.quotes.identifiers = '`'; templates.quotes.escape = '``'; From 752ac4b82a5fb137efb51d1523c812dffdcc7b70 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 30 Jan 2025 13:42:33 +0200 Subject: [PATCH 2/2] add tests --- rust/cubesql/cubesql/src/compile/mod.rs | 51 +++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/rust/cubesql/cubesql/src/compile/mod.rs b/rust/cubesql/cubesql/src/compile/mod.rs index 2e99f16eb60d4..36ea607b8f932 100644 --- a/rust/cubesql/cubesql/src/compile/mod.rs +++ b/rust/cubesql/cubesql/src/compile/mod.rs @@ -15945,6 +15945,57 @@ LIMIT {{ limit }}{% endif %}"#.to_string(), assert!(sql.contains(" AS DECIMAL(38,10))")); } + #[tokio::test] + async fn test_extract_epoch_pushdown() { + if !Rewriter::sql_push_down_enabled() { + return; + } + init_testing_logger(); + + let query = " + SELECT LOWER(customer_gender), + MAX(CAST(FLOOR(EXTRACT(EPOCH FROM order_date) / 31536000) AS bigint)) AS max_years + FROM KibanaSampleDataEcommerce + GROUP BY 1 + "; + + // Generic + let query_plan = + convert_select_to_query_plan(query.to_string(), DatabaseProtocol::PostgreSQL).await; + + let physical_plan = query_plan.as_physical_plan().await.unwrap(); + println!( + "Physical plan: {}", + displayable(physical_plan.as_ref()).indent() + ); + + let logical_plan = query_plan.as_logical_plan(); + let sql = logical_plan.find_cube_scan_wrapped_sql().wrapped_sql.sql; + assert!(sql.contains("EXTRACT(EPOCH")); + + // Databricks + let query_plan = convert_select_to_query_plan_customized( + query.to_string(), + DatabaseProtocol::PostgreSQL, + vec![ + ("expressions/timestamp_literal".to_string(), "from_utc_timestamp('{{ value }}', 'UTC')".to_string()), + ("expressions/extract".to_string(), "{% if date_part|lower == \"epoch\" %}unix_timestamp({{ expr }}){% else %}EXTRACT({{ date_part }} FROM {{ expr }}){% endif %}".to_string()), + ], + ) + .await; + + let physical_plan = query_plan.as_physical_plan().await.unwrap(); + println!( + "Physical plan: {}", + displayable(physical_plan.as_ref()).indent() + ); + + let logical_plan = query_plan.as_logical_plan(); + let sql = logical_plan.find_cube_scan_wrapped_sql().wrapped_sql.sql; + assert!(!sql.contains("EXTRACT(EPOCH")); + assert!(sql.contains("unix_timestamp")); + } + #[tokio::test] async fn test_push_down_to_grouped_query_with_filters() { if !Rewriter::sql_push_down_enabled() {