From df31e8958eccfca893723edb5a4f5c036ef62c94 Mon Sep 17 00:00:00 2001 From: Alexandr Romanenko Date: Mon, 20 Jan 2025 15:49:03 +0100 Subject: [PATCH] update --- .../src/cube_bridge/measure_definition.rs | 3 + .../cubesqlplanner/src/plan/join.rs | 77 ++++++++++++- .../cubesqlplanner/src/plan/mod.rs | 2 +- .../cubesqlplanner/src/planner/base_member.rs | 4 + .../cubesqlplanner/src/planner/base_query.rs | 2 +- .../src/planner/filter/base_filter.rs | 82 ++++++++++---- .../src/planner/filter/filter_operator.rs | 3 +- .../planners/dimension_subquery_planner.rs | 2 +- .../planners/multi_stage/applied_state.rs | 42 ++++--- .../planner/planners/multi_stage/member.rs | 65 +++++++++-- .../multi_stage/member_query_planner.rs | 97 ++++++++-------- .../planners/multi_stage_query_planner.rs | 107 +++++++++++++----- .../multiplied_measures_query_planner.rs | 21 +++- .../src/planner/planners/query_planner.rs | 29 +++-- .../src/planner/query_properties.rs | 8 +- 15 files changed, 400 insertions(+), 144 deletions(-) diff --git a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/measure_definition.rs b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/measure_definition.rs index 8172b846ffe74..e147ec65bea03 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/measure_definition.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/measure_definition.rs @@ -26,6 +26,9 @@ pub struct RollingWindow { pub trailing: Option, pub leading: Option, pub offset: Option, + #[serde(rename = "type")] + pub rolling_type: Option, + pub granularity: Option, } #[derive(Serialize, Deserialize, Debug)] diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs index a7b24d17b1dba..818cbfed95865 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/join.rs @@ -1,4 +1,5 @@ use super::{Expr, SingleAliasedSource}; +use crate::planner::query_tools::QueryTools; use crate::planner::sql_templates::PlanSqlTemplates; use crate::planner::{BaseJoinCondition, VisitorContext}; use cubenativeutils::CubeError; @@ -6,7 +7,7 @@ use lazy_static::lazy_static; use std::rc::Rc; -pub struct RollingWindowJoinCondition { +pub struct RegularRollingWindowJoinCondition { time_series_source: String, trailing_interval: Option, leading_interval: Option, @@ -14,7 +15,7 @@ pub struct RollingWindowJoinCondition { time_dimension: Expr, } -impl RollingWindowJoinCondition { +impl RegularRollingWindowJoinCondition { pub fn new( time_series_source: String, trailing_interval: Option, @@ -87,6 +88,50 @@ impl RollingWindowJoinCondition { } } +pub struct ToDateRollingWindowJoinCondition { + time_series_source: String, + granularity: String, + time_dimension: Expr, + query_tools: Rc, +} + +impl ToDateRollingWindowJoinCondition { + pub fn new( + time_series_source: String, + granularity: String, + time_dimension: Expr, + query_tools: Rc, + ) -> Self { + Self { + time_series_source, + granularity, + time_dimension, + query_tools, + } + } + + pub fn to_sql( + &self, + templates: &PlanSqlTemplates, + context: Rc, + ) -> Result { + let date_column = self.time_dimension.to_sql(templates, context)?; + + //(dateFrom, dateTo, dateField, dimensionDateFrom, dimensionDateTo, isFromStartToEnd) => `${dateField} >= ${this.timeGroupedColumn(granularity, dateFrom)} AND ${dateField} <= ${dateTo}` + + let date_from = + templates.column_reference(&Some(self.time_series_source.clone()), "date_to")?; + let date_to = + templates.column_reference(&Some(self.time_series_source.clone()), "date_from")?; + let grouped_from = self + .query_tools + .base_tools() + .time_grouped_column(self.granularity.clone(), date_from)?; + let result = format!("{date_column} >= {grouped_from} and {date_column} <= {date_to}"); + Ok(result) + } +} + pub struct DimensionJoinCondition { // AND (... OR ...) conditions: Vec>, @@ -145,7 +190,8 @@ impl DimensionJoinCondition { pub enum JoinCondition { DimensionJoinCondition(DimensionJoinCondition), BaseJoinCondition(Rc), - RollingWindowJoinCondition(RollingWindowJoinCondition), + RegularRollingWindowJoinCondition(RegularRollingWindowJoinCondition), + ToDateRollingWindowJoinCondition(ToDateRollingWindowJoinCondition), } impl JoinCondition { @@ -153,14 +199,14 @@ impl JoinCondition { Self::DimensionJoinCondition(DimensionJoinCondition::new(conditions, null_check)) } - pub fn new_rolling_join( + pub fn new_regular_rolling_join( time_series_source: String, trailing_interval: Option, leading_interval: Option, offset: String, time_dimension: Expr, ) -> Self { - Self::RollingWindowJoinCondition(RollingWindowJoinCondition::new( + Self::RegularRollingWindowJoinCondition(RegularRollingWindowJoinCondition::new( time_series_source, trailing_interval, leading_interval, @@ -169,6 +215,20 @@ impl JoinCondition { )) } + pub fn new_to_date_rolling_join( + time_series_source: String, + granularity: String, + time_dimension: Expr, + query_tools: Rc, + ) -> Self { + Self::ToDateRollingWindowJoinCondition(ToDateRollingWindowJoinCondition::new( + time_series_source, + granularity, + time_dimension, + query_tools, + )) + } + pub fn new_base_join(base: Rc) -> Self { Self::BaseJoinCondition(base) } @@ -181,7 +241,12 @@ impl JoinCondition { match &self { JoinCondition::DimensionJoinCondition(cond) => cond.to_sql(templates, context), JoinCondition::BaseJoinCondition(cond) => cond.to_sql(context, templates), - JoinCondition::RollingWindowJoinCondition(cond) => cond.to_sql(templates, context), + JoinCondition::RegularRollingWindowJoinCondition(cond) => { + cond.to_sql(templates, context) + } + JoinCondition::ToDateRollingWindowJoinCondition(cond) => { + cond.to_sql(templates, context) + } } } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/mod.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/mod.rs index 3799935638bd2..2117bf42d5611 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/mod.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/mod.rs @@ -16,7 +16,7 @@ pub use cte::Cte; pub use expression::{Expr, MemberExpression}; pub use filter::{Filter, FilterGroup, FilterItem}; pub use from::{From, FromSource, SingleAliasedSource, SingleSource}; -pub use join::{Join, JoinCondition, JoinItem, RollingWindowJoinCondition}; +pub use join::{Join, JoinCondition, JoinItem, RegularRollingWindowJoinCondition}; pub use order::OrderBy; pub use query_plan::QueryPlan; pub use schema::{QualifiedColumnName, Schema, SchemaColumn}; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/base_member.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/base_member.rs index 394b2a71f2994..47f24a93ba00c 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/base_member.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/base_member.rs @@ -44,6 +44,10 @@ impl BaseMemberHelper { members.iter().map(|m| m.alias_name()).collect_vec() } + pub fn extract_symbols_from_members(members: &Vec>) -> Vec> { + members.iter().map(|m| m.member_evaluator()).collect_vec() + } + pub fn default_alias( cube_name: &String, member_name: &String, diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs index 1b1613e77d725..d9cc10ed41f40 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs @@ -41,7 +41,7 @@ impl BaseQuery { pub fn build_sql_and_params(&self) -> Result, CubeError> { let templates = PlanSqlTemplates::new(self.query_tools.templates_render()); let query_planner = QueryPlanner::new(self.request.clone(), self.query_tools.clone()); - let plan = query_planner.build_sql()?; + let plan = query_planner.plan()?; let sql = plan.to_sql(&templates)?; let (result_sql, params) = self.query_tools.build_sql_and_params(&sql, true)?; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs index fa3ccfdcae26e..4b77c9eef4e96 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs @@ -114,7 +114,12 @@ impl BaseFilter { FilterOperator::Equal => self.equals_where(&member_sql)?, FilterOperator::NotEqual => self.not_equals_where(&member_sql)?, FilterOperator::InDateRange => self.in_date_range(&member_sql)?, - FilterOperator::InDateRangeExtended => self.in_date_range_extended(&member_sql)?, + FilterOperator::RegularRollingWindowDateRange => { + self.regular_rolling_window_date_range(&member_sql)? + } + FilterOperator::ToDateRollingWindowDateRange => { + self.to_date_rolling_window_date_range(&member_sql)? + } FilterOperator::In => self.in_where(&member_sql)?, FilterOperator::NotIn => self.not_in_where(&member_sql)?, FilterOperator::Set => self.set_where(&member_sql)?, @@ -211,7 +216,7 @@ impl BaseFilter { } fn in_date_range(&self, member_sql: &str) -> Result { - let (from, to) = self.allocate_date_params()?; + let (from, to) = self.allocate_date_params(true)?; self.templates .time_range_filter(member_sql.to_string(), from, to) } @@ -236,8 +241,9 @@ impl BaseFilter { Ok(date.to_string()) } } - fn in_date_range_extended(&self, member_sql: &str) -> Result { - let (from, to) = self.allocate_date_params()?; + + fn regular_rolling_window_date_range(&self, member_sql: &str) -> Result { + let (from, to) = self.allocate_date_params(false)?; let from = if self.values.len() >= 3 { self.extend_date_range_bound(from, &self.values[2], true)? @@ -251,8 +257,37 @@ impl BaseFilter { to }; - self.templates - .time_range_filter(member_sql.to_string(), from, to) + let date_field = self + .query_tools + .base_tools() + .convert_tz(member_sql.to_string())?; + self.templates.time_range_filter(date_field, from, to) + } + + fn to_date_rolling_window_date_range(&self, member_sql: &str) -> Result { + let (from, to) = self.allocate_date_params(false)?; + + let from = if self.values.len() >= 3 { + if let Some(granularity) = &self.values[2] { + self.query_tools + .base_tools() + .time_grouped_column(granularity.clone(), from)? + } else { + return Err(CubeError::user(format!( + "Granularity required for to_date rolling window" + ))); + } + } else { + return Err(CubeError::user(format!( + "Granularity required for to_date rolling window" + ))); + }; + + let date_field = self + .query_tools + .base_tools() + .convert_tz(member_sql.to_string())?; + self.templates.time_range_filter(date_field, from, to) } fn in_where(&self, member_sql: &str) -> Result { @@ -353,12 +388,16 @@ impl BaseFilter { )) } - fn allocate_date_params(&self) -> Result<(String, String), CubeError> { + fn allocate_date_params(&self, use_db_time_zone: bool) -> Result<(String, String), CubeError> { if self.values.len() >= 2 { let from = if let Some(from_str) = &self.values[0] { - self.query_tools - .base_tools() - .in_db_time_zone(self.format_from_date(&from_str)?)? + let from = self.format_from_date(&from_str)?; + + if use_db_time_zone { + self.query_tools.base_tools().in_db_time_zone(from)? + } else { + from + } } else { return Err(CubeError::user(format!( "Arguments for date range is not valid" @@ -366,9 +405,13 @@ impl BaseFilter { }; let to = if let Some(to_str) = &self.values[1] { - self.query_tools - .base_tools() - .in_db_time_zone(self.format_to_date(&to_str)?)? + let to = self.format_to_date(&to_str)?; + + if use_db_time_zone { + self.query_tools.base_tools().in_db_time_zone(to)? + } else { + to + } } else { return Err(CubeError::user(format!( "Arguments for date range is not valid" @@ -411,11 +454,7 @@ impl BaseFilter { "0".repeat(precision as usize) )); } - //FIXME chrono don't support parsing date without specified format - Err(CubeError::user(format!( - "Unsupported date format: {}", - date - ))) + Ok(date.to_string()) } fn format_to_date(&self, date: &str) -> Result { @@ -447,11 +486,8 @@ impl BaseFilter { "9".repeat(precision as usize) )); } - //FIXME chrono don't support parsing date without specified format - Err(CubeError::user(format!( - "Unsupported date format: {}", - date - ))) + + Ok(date.to_string()) } fn allocate_param(&self, param: &str) -> String { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/filter_operator.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/filter_operator.rs index e0865b76c0c44..99d946e462b98 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/filter_operator.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/filter_operator.rs @@ -6,7 +6,8 @@ pub enum FilterOperator { Equal, NotEqual, InDateRange, - InDateRangeExtended, + RegularRollingWindowDateRange, + ToDateRollingWindowDateRange, In, NotIn, Set, diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/dimension_subquery_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/dimension_subquery_planner.rs index 5b6fee5cdd235..7b08972b1c8c2 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/dimension_subquery_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/dimension_subquery_planner.rs @@ -125,7 +125,7 @@ impl DimensionSubqueryPlanner { false, )?; let query_planner = QueryPlanner::new(sub_query_properties, self.query_tools.clone()); - let sub_query = query_planner.build_sql()?; + let sub_query = query_planner.plan()?; let sub_query_alias = format!("{cube_name}_{dim_name}_subquery"); let conditions = primary_keys_dimensions diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs index 7a4061c43f64d..a60b755293819 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs @@ -160,26 +160,43 @@ impl MultiStageAppliedState { false } - pub fn expand_date_range_filter( + pub fn replace_regular_date_range_filter( &mut self, member_name: &String, left_interval: Option, right_interval: Option, ) { - self.time_dimensions_filters = self.expand_date_range_filter_impl( + let operator = FilterOperator::RegularRollingWindowDateRange; + let values = vec![left_interval.clone(), right_interval.clone()]; + self.time_dimensions_filters = self.change_date_range_filter_impl( member_name, &self.time_dimensions_filters, - &left_interval, - &right_interval, + &operator, + &values, ); } - fn expand_date_range_filter_impl( + pub fn replace_to_date_date_range_filter( + &mut self, + member_name: &String, + granularity: &String, + ) { + let operator = FilterOperator::ToDateRollingWindowDateRange; + let values = vec![Some(granularity.clone())]; + self.time_dimensions_filters = self.change_date_range_filter_impl( + member_name, + &self.time_dimensions_filters, + &operator, + &values, + ); + } + + fn change_date_range_filter_impl( &self, member_name: &String, filters: &Vec, - left_interval: &Option, - right_interval: &Option, + operator: &FilterOperator, + additional_values: &Vec>, ) -> Vec { let mut result = Vec::new(); for item in filters.iter() { @@ -187,11 +204,11 @@ impl MultiStageAppliedState { FilterItem::Group(group) => { let new_group = FilterItem::Group(Rc::new(FilterGroup::new( group.operator.clone(), - self.expand_date_range_filter_impl( + self.change_date_range_filter_impl( member_name, filters, - left_interval, - right_interval, + operator, + additional_values, ), ))); result.push(new_group); @@ -201,9 +218,8 @@ impl MultiStageAppliedState { && matches!(itm.filter_operator(), FilterOperator::InDateRange) { let mut values = itm.values().clone(); - values.push(left_interval.clone()); - values.push(right_interval.clone()); - itm.change_operator(FilterOperator::InDateRangeExtended, values) + values.extend(additional_values.iter().cloned()); + itm.change_operator(operator.clone(), values) } else { itm.clone() }; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member.rs index e7e97dc849ff3..1801c2538e3d8 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member.rs @@ -66,13 +66,55 @@ pub enum MultiStageLeafMemberType { } #[derive(Clone)] -pub struct RollingWindowDescription { - pub time_dimension: Rc, +pub struct RegularRollingWindow { pub trailing: Option, pub leading: Option, pub offset: String, } +#[derive(Clone)] +pub struct ToDateRollingWindow { + pub granularity: String, +} + +#[derive(Clone)] +pub enum RollingWindowType { + Regular(RegularRollingWindow), + ToDate(ToDateRollingWindow), +} + +#[derive(Clone)] +pub struct RollingWindowDescription { + pub time_dimension: Rc, + pub rolling_window: RollingWindowType, +} + +impl RollingWindowDescription { + pub fn new_regular( + time_dimension: Rc, + trailing: Option, + leading: Option, + offset: String, + ) -> Self { + let regular_window = RegularRollingWindow { + trailing, + leading, + offset, + }; + Self { + time_dimension, + rolling_window: RollingWindowType::Regular(regular_window), + } + } + + pub fn new_to_date(time_dimension: Rc, granularity: String) -> Self { + Self { + time_dimension, + rolling_window: RollingWindowType::ToDate(ToDateRollingWindow { granularity }), + } + } +} + #[derive(Clone)] pub struct RunningTotalDescription { pub time_dimension: Rc, @@ -94,7 +136,6 @@ pub struct MultiStageInodeMember { add_group_by: Vec, group_by: Option>, time_shifts: Vec, - is_ungrupped: bool, } impl MultiStageInodeMember { @@ -104,7 +145,6 @@ impl MultiStageInodeMember { add_group_by: Vec, group_by: Option>, time_shifts: Vec, - is_ungrupped: bool, ) -> Self { Self { inode_type, @@ -112,7 +152,6 @@ impl MultiStageInodeMember { add_group_by, group_by, time_shifts, - is_ungrupped, } } @@ -135,10 +174,6 @@ impl MultiStageInodeMember { pub fn time_shifts(&self) -> &Vec { &self.time_shifts } - - pub fn is_ungrupped(&self) -> bool { - self.is_ungrupped - } } #[derive(Clone)] @@ -150,13 +185,19 @@ pub enum MultiStageMemberType { pub struct MultiStageMember { member_type: MultiStageMemberType, evaluation_node: Rc, + is_ungrupped: bool, } impl MultiStageMember { - pub fn new(member_type: MultiStageMemberType, evaluation_node: Rc) -> Rc { + pub fn new( + member_type: MultiStageMemberType, + evaluation_node: Rc, + is_ungrupped: bool, + ) -> Rc { Rc::new(Self { member_type, evaluation_node, + is_ungrupped, }) } @@ -171,4 +212,8 @@ impl MultiStageMember { pub fn full_name(&self) -> String { self.evaluation_node.full_name() } + + pub fn is_ungrupped(&self) -> bool { + self.is_ungrupped + } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs index 06fee5c058ecb..b717371719ce0 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs @@ -6,13 +6,10 @@ use crate::plan::{ Cte, Expr, From, JoinBuilder, JoinCondition, MemberExpression, OrderBy, QualifiedColumnName, QueryPlan, Schema, SelectBuilder, TimeSeries, }; -use crate::planner::planners::{ - FullKeyAggregateQueryPlanner, MultipliedMeasuresQueryPlanner, OrderPlanner, SimpleQueryPlanner, -}; +use crate::planner::planners::{multi_stage::RollingWindowType, OrderPlanner, QueryPlanner}; use crate::planner::query_tools::QueryTools; use crate::planner::sql_evaluator::sql_nodes::SqlNodesFactory; use crate::planner::sql_evaluator::ReferencesBuilder; -use crate::planner::sql_templates::PlanSqlTemplates; use crate::planner::QueryProperties; use crate::planner::{BaseDimension, BaseMeasure, BaseMember, BaseMemberHelper, BaseTimeDimension}; use cubenativeutils::CubeError; @@ -105,16 +102,31 @@ impl MultiStageMemberQueryPlanner { let rolling_base_cte_schema = cte_schemas.get(input).unwrap().clone(); let time_dimension_alias = rolling_base_cte_schema.resolve_member_alias(&rolling_window_desc.time_dimension); - let on = JoinCondition::new_rolling_join( - root_alias.clone(), - rolling_window_desc.trailing.clone(), - rolling_window_desc.leading.clone(), - rolling_window_desc.offset.clone(), - Expr::Reference(QualifiedColumnName::new( - Some(alias.clone()), - time_dimension_alias, - )), - ); + let on = match &rolling_window_desc.rolling_window { + RollingWindowType::Regular(regular_rolling_window) => { + JoinCondition::new_regular_rolling_join( + root_alias.clone(), + regular_rolling_window.trailing.clone(), + regular_rolling_window.leading.clone(), + regular_rolling_window.offset.clone(), + Expr::Reference(QualifiedColumnName::new( + Some(alias.clone()), + time_dimension_alias, + )), + ) + } + RollingWindowType::ToDate(to_date_rolling_window) => { + JoinCondition::new_to_date_rolling_join( + root_alias.clone(), + to_date_rolling_window.granularity.clone(), + Expr::Reference(QualifiedColumnName::new( + Some(alias.clone()), + time_dimension_alias, + )), + self.query_tools.clone(), + ) + } + }; join_builder.left_join_table_reference( input.clone(), rolling_base_cte_schema.clone(), @@ -124,14 +136,22 @@ impl MultiStageMemberQueryPlanner { let from = From::new_from_join(join_builder.build()); - let group_by = dimensions - .iter() - .map(|dim| Expr::Member(MemberExpression::new(dim.clone()))) - .collect_vec(); + let group_by = if self.description.member().is_ungrupped() { + vec![] + } else { + dimensions + .iter() + .map(|dim| Expr::Member(MemberExpression::new(dim.clone()))) + .collect_vec() + }; let mut context_factory = SqlNodesFactory::new(); context_factory.set_rolling_window(true); + if self.description.member().is_ungrupped() { + context_factory.set_ungrouped(true); + } + let references_builder = ReferencesBuilder::new(from.clone()); let mut render_references = HashMap::new(); let mut select_builder = SelectBuilder::new(from.clone()); @@ -186,7 +206,7 @@ impl MultiStageMemberQueryPlanner { format!("{}_join", self.description.alias()), ); - let group_by = if multi_stage_member.is_ungrupped() { + let group_by = if self.description.member().is_ungrupped() { vec![] } else { dimensions @@ -195,7 +215,7 @@ impl MultiStageMemberQueryPlanner { .collect_vec() }; - let order_by = if multi_stage_member.is_ungrupped() { + let order_by = if self.description.member().is_ungrupped() { vec![] } else { self.query_order()? @@ -243,6 +263,9 @@ impl MultiStageMemberQueryPlanner { select_builder.set_group_by(group_by); select_builder.set_order_by(order_by); context_factory.set_render_references(render_references); + if self.description.member().is_ungrupped() { + context_factory.set_ungrouped(true); + } let select = select_builder.build(context_factory); Ok(Rc::new(Cte::new_from_select( @@ -356,43 +379,27 @@ impl MultiStageMemberQueryPlanner { None, None, true, - false, + self.description.member().is_ungrupped(), )?; let mut node_factory = SqlNodesFactory::new(); node_factory.set_time_shifts(self.description.state().time_shifts().clone()); - /* if cte_query_properties + if cte_query_properties .full_key_aggregate_measures()? .has_multi_stage_measures() { return Err(CubeError::internal(format!( "Leaf multi stage query cannot contain multi stage member" ))); - } */ + } - let cte_select = if cte_query_properties.is_simple_query()? { - let planner = SimpleQueryPlanner::new( - self.query_tools.clone(), - cte_query_properties.clone(), - node_factory.clone(), - ); - planner.plan()? - } else { - let multiplied_measures_query_planner = MultipliedMeasuresQueryPlanner::try_new( - self.query_tools.clone(), - cte_query_properties.clone(), - node_factory.clone(), - )?; - let full_key_aggregate_planner = FullKeyAggregateQueryPlanner::new( - cte_query_properties.clone(), - node_factory.clone(), - PlanSqlTemplates::new(self.query_tools.templates_render()), - ); - let subqueries = multiplied_measures_query_planner.plan_queries()?; - let result = full_key_aggregate_planner.plan(subqueries, vec![])?; - result - }; + let query_planner = QueryPlanner::new_with_context_factory( + cte_query_properties.clone(), + self.query_tools.clone(), + node_factory, + ); + let cte_select = query_planner.plan()?; let result = Cte::new_from_select(cte_select, self.description.alias().clone()); Ok(Rc::new(result)) } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage_query_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage_query_planner.rs index 1a2f2955d71b8..6404ca37b469d 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage_query_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage_query_planner.rs @@ -106,7 +106,7 @@ impl MultiStageQueryPlanner { fn create_multi_stage_inode_member( &self, base_member: Rc, - ) -> Result { + ) -> Result<(MultiStageInodeMember, bool), CubeError> { let inode = if let Some(measure) = BaseMeasure::try_new(base_member.clone(), self.query_tools.clone())? { @@ -129,24 +129,28 @@ impl MultiStageQueryPlanner { }; let is_ungrupped = match &member_type { MultiStageInodeMemberType::Rank | MultiStageInodeMemberType::Calculate => true, - _ => false, + _ => self.query_properties.ungrouped(), }; - MultiStageInodeMember::new( - member_type, - measure.reduce_by().clone().unwrap_or_default(), - measure.add_group_by().clone().unwrap_or_default(), - measure.group_by().clone(), - time_shifts, + ( + MultiStageInodeMember::new( + member_type, + measure.reduce_by().clone().unwrap_or_default(), + measure.add_group_by().clone().unwrap_or_default(), + measure.group_by().clone(), + time_shifts, + ), is_ungrupped, ) } else { - MultiStageInodeMember::new( - MultiStageInodeMemberType::Calculate, - vec![], - vec![], - None, - vec![], - false, + ( + MultiStageInodeMember::new( + MultiStageInodeMemberType::Calculate, + vec![], + vec![], + None, + vec![], + ), + self.query_properties.ungrouped(), ) }; Ok(inode) @@ -168,6 +172,7 @@ impl MultiStageQueryPlanner { time_dimension.clone(), )), time_dimension.member_evaluator(), + true, ), state.clone(), vec![], @@ -190,6 +195,7 @@ impl MultiStageQueryPlanner { MultiStageMember::new( MultiStageMemberType::Leaf(MultiStageLeafMemberType::Measure), member, + self.query_properties.ungrouped(), ), state, vec![], @@ -199,6 +205,28 @@ impl MultiStageQueryPlanner { Ok(description) } + fn get_to_date_rolling_granularity( + &self, + rolling_window: &RollingWindow, + ) -> Result, CubeError> { + let is_to_date = rolling_window + .rolling_type + .as_ref() + .map_or(false, |tp| tp == "to_date"); + + if is_to_date { + if let Some(granularity) = &rolling_window.granularity { + Ok(Some(granularity.clone())) + } else { + Err(CubeError::user(format!( + "Granularity required for to_date rolling window" + ))) + } + } else { + Ok(None) + } + } + fn make_rolling_base_state( &self, time_dimension: Rc, @@ -220,11 +248,15 @@ impl MultiStageQueryPlanner { new_state.change_time_dimension_granularity(&time_dimension_name, result_granularity); - new_state.expand_date_range_filter( - &time_dimension_name, - rolling_window.trailing.clone(), - rolling_window.leading.clone(), - ); + if let Some(granularity) = self.get_to_date_rolling_granularity(rolling_window)? { + new_state.replace_to_date_date_range_filter(&time_dimension_name, &granularity); + } else { + new_state.replace_regular_date_range_filter( + &time_dimension_name, + rolling_window.trailing.clone(), + rolling_window.leading.clone(), + ); + } Ok(Rc::new(new_state)) } @@ -244,6 +276,8 @@ impl MultiStageQueryPlanner { trailing: Some("unbounded".to_string()), leading: None, offset: None, + rolling_type: None, + granularity: None, } }; let time_dimensions = self.query_properties.time_dimensions(); @@ -276,11 +310,17 @@ impl MultiStageQueryPlanner { let alias = format!("cte_{}", descriptions.len()); - let rolling_window_descr = RollingWindowDescription { - time_dimension: time_dimension.clone(), - trailing: rolling_window.trailing.clone(), - leading: rolling_window.leading.clone(), - offset: rolling_window.offset.clone().unwrap_or("end".to_string()), + let rolling_window_descr = if let Some(granularity) = + self.get_to_date_rolling_granularity(&rolling_window)? + { + RollingWindowDescription::new_to_date(time_dimension, granularity) + } else { + RollingWindowDescription::new_regular( + time_dimension, + rolling_window.trailing.clone(), + rolling_window.leading.clone(), + rolling_window.offset.clone().unwrap_or("end".to_string()), + ) }; let inode_member = MultiStageInodeMember::new( @@ -289,11 +329,14 @@ impl MultiStageQueryPlanner { vec![], None, vec![], - false, ); let description = MultiStageQueryDescription::new( - MultiStageMember::new(MultiStageMemberType::Inode(inode_member), member), + MultiStageMember::new( + MultiStageMemberType::Inode(inode_member), + member, + self.query_properties.ungrouped(), + ), state.clone(), input, alias.clone(), @@ -342,13 +385,15 @@ impl MultiStageQueryPlanner { MultiStageMember::new( MultiStageMemberType::Leaf(MultiStageLeafMemberType::Measure), member.clone(), + self.query_properties.ungrouped(), ), state.clone(), vec![], alias.clone(), ) } else { - let multi_stage_member = self.create_multi_stage_inode_member(member.clone())?; + let (multi_stage_member, is_ungrupped) = + self.create_multi_stage_inode_member(member.clone())?; let dimensions_to_add = multi_stage_member .add_group_by() @@ -386,7 +431,11 @@ impl MultiStageQueryPlanner { let alias = format!("cte_{}", descriptions.len()); MultiStageQueryDescription::new( - MultiStageMember::new(MultiStageMemberType::Inode(multi_stage_member), member), + MultiStageMember::new( + MultiStageMemberType::Inode(multi_stage_member), + member, + is_ungrupped, + ), state.clone(), input, alias.clone(), diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multiplied_measures_query_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multiplied_measures_query_planner.rs index 65ad3d132466e..cce7bfa7c2aa1 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multiplied_measures_query_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multiplied_measures_query_planner.rs @@ -361,8 +361,25 @@ impl MultipliedMeasuresQueryPlanner { .query_properties .dimensions_for_select_append(dimensions); - let subquery_dimensions = - collect_sub_query_dimensions_from_members(&dimensions, self.query_tools.clone())?; + let mut symbols_for_subquery_dimensions = + BaseMemberHelper::extract_symbols_from_members(&dimensions); + for item in self.query_properties.dimensions_filters() { + item.find_all_member_evaluators(&mut symbols_for_subquery_dimensions); + } + + for item in self.query_properties.measures_filters() { + item.find_all_member_evaluators(&mut symbols_for_subquery_dimensions); + } + + let symbols_for_subquery_dimensions = symbols_for_subquery_dimensions + .into_iter() + .unique_by(|m| m.full_name()) + .collect_vec(); + + let subquery_dimensions = collect_sub_query_dimensions_from_symbols( + &symbols_for_subquery_dimensions, + self.query_tools.clone(), + )?; let dimension_subquery_planner = DimensionSubqueryPlanner::try_new( &subquery_dimensions, diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/query_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/query_planner.rs index 954a599181e2c..5780daf4e73bc 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/query_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/query_planner.rs @@ -13,6 +13,7 @@ use std::rc::Rc; pub struct QueryPlanner { query_tools: Rc, request: Rc, + context_factory: Option, } impl QueryPlanner { @@ -20,19 +21,33 @@ impl QueryPlanner { Self { request, query_tools, + context_factory: None, } } - pub fn build_sql(&self) -> Result, CubeError> { + pub fn new_with_context_factory( + request: Rc, + query_tools: Rc, + context_factory: SqlNodesFactory, + ) -> Self { + Self { + request, + query_tools, + context_factory: Some(context_factory), + } + } + + pub fn plan(&self) -> Result, CubeError> { let templates = PlanSqlTemplates::new(self.query_tools.templates_render()); - self.build_sql_and_params_impl(templates) + self.build_sql_impl(templates) } - fn build_sql_and_params_impl( - &self, - templates: PlanSqlTemplates, - ) -> Result, CubeError> { - let mut nodes_factory = SqlNodesFactory::new(); + fn build_sql_impl(&self, templates: PlanSqlTemplates) -> Result, CubeError> { + let mut nodes_factory = if let Some(context_factory) = &self.context_factory { + context_factory.clone() + } else { + SqlNodesFactory::new() + }; if self.request.ungrouped() { nodes_factory.set_ungrouped(true) diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs index d6b07304ae690..02bcc14e09579 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs @@ -461,11 +461,9 @@ impl QueryProperties { } } pub fn all_member_symbols(&self, exclude_time_dimensions: bool) -> Vec> { - let mut members = self - .all_members(exclude_time_dimensions) - .into_iter() - .map(|m| m.member_evaluator()) - .collect_vec(); + let mut members = BaseMemberHelper::extract_symbols_from_members( + &self.all_members(exclude_time_dimensions), + ); for filter_item in self.dimensions_filters.iter() { filter_item.find_all_member_evaluators(&mut members); }