From be140ecdd0425f6ea3420520649f68ff81ae46f1 Mon Sep 17 00:00:00 2001 From: Mikhail Cheshkov Date: Thu, 23 Jan 2025 20:20:58 +0200 Subject: [PATCH 1/5] feat(cubesql): Support %s in format (#9129) Used by Metabase introspection See https://github.com/metabase/metabase/pull/50900 Fixes #9126 --- .../cubesql/src/compile/engine/udf/common.rs | 36 ++ rust/cubesql/cubesql/src/compile/mod.rs | 24 ++ ...tests__formatted_null_string_is_empty.snap | 9 + ...sql__compile__tests__formatted_string.snap | 9 + ...ql__compile__tests__formatted_strings.snap | 9 + ...__test_metabase_v0_51_8_introspection.snap | 308 ++++++++++++++++++ .../src/compile/test/test_introspection.rs | 89 +++++ 7 files changed, 484 insertions(+) create mode 100644 rust/cubesql/cubesql/src/compile/snapshots/cubesql__compile__tests__formatted_null_string_is_empty.snap create mode 100644 rust/cubesql/cubesql/src/compile/snapshots/cubesql__compile__tests__formatted_string.snap create mode 100644 rust/cubesql/cubesql/src/compile/snapshots/cubesql__compile__tests__formatted_strings.snap create mode 100644 rust/cubesql/cubesql/src/compile/test/snapshots/cubesql__compile__test__test_introspection__test_metabase_v0_51_8_introspection.snap diff --git a/rust/cubesql/cubesql/src/compile/engine/udf/common.rs b/rust/cubesql/cubesql/src/compile/engine/udf/common.rs index c31af0d18db8a..127fed402cc73 100644 --- a/rust/cubesql/cubesql/src/compile/engine/udf/common.rs +++ b/rust/cubesql/cubesql/src/compile/engine/udf/common.rs @@ -3281,6 +3281,42 @@ pub fn create_format_udf() -> ScalarUDF { // %% is escaped to single % result.push('%'); } + Some('s') => { + // Handle %s - regular string + if arg_index >= args.len() { + return Err(DataFusionError::Execution( + "Not enough arguments for format string".to_string(), + )); + } + + let arg = &args[arg_index]; + let value = match arg.data_type() { + DataType::Utf8 => { + let str_arr = downcast_string_arg!(arg, "arg", i32); + if str_arr.is_null(i) { + // A null value is treated as an empty string + String::new() + } else { + str_arr.value(i).to_string() + } + } + _ => { + // For other types, try to convert to string + let str_arr = cast(&arg, &DataType::Utf8)?; + let str_arr = + str_arr.as_any().downcast_ref::().unwrap(); + if str_arr.is_null(i) { + // A null value is treated as an empty string + String::new() + } else { + str_arr.value(i).to_string() + } + } + }; + + result.push_str(&value); + arg_index += 1; + } Some(c) => { return Err(DataFusionError::Execution(format!( "Unsupported format specifier %{}", diff --git a/rust/cubesql/cubesql/src/compile/mod.rs b/rust/cubesql/cubesql/src/compile/mod.rs index 22c588f299d9c..2e99f16eb60d4 100644 --- a/rust/cubesql/cubesql/src/compile/mod.rs +++ b/rust/cubesql/cubesql/src/compile/mod.rs @@ -16214,6 +16214,30 @@ LIMIT {{ limit }}{% endif %}"#.to_string(), #[tokio::test] async fn test_format_function() -> Result<(), CubeError> { + // Test: Basic usage with a single string + let result = execute_query( + "SELECT format('%s', 'foo') AS formatted_string".to_string(), + DatabaseProtocol::PostgreSQL, + ) + .await?; + insta::assert_snapshot!("formatted_string", result); + + // Test: Basic usage with a single null string + let result = execute_query( + "SELECT format('%s', NULL) = '' AS formatted_null_string_is_empty".to_string(), + DatabaseProtocol::PostgreSQL, + ) + .await?; + insta::assert_snapshot!("formatted_null_string_is_empty", result); + + // Test: Basic usage with a multiple strings + let result = execute_query( + "SELECT format('%s.%s', 'foo', 'bar') AS formatted_strings".to_string(), + DatabaseProtocol::PostgreSQL, + ) + .await?; + insta::assert_snapshot!("formatted_strings", result); + // Test: Basic usage with a single identifier let result = execute_query( "SELECT format('%I', 'column_name') AS formatted_identifier".to_string(), diff --git a/rust/cubesql/cubesql/src/compile/snapshots/cubesql__compile__tests__formatted_null_string_is_empty.snap b/rust/cubesql/cubesql/src/compile/snapshots/cubesql__compile__tests__formatted_null_string_is_empty.snap new file mode 100644 index 0000000000000..9d1498ea38d5a --- /dev/null +++ b/rust/cubesql/cubesql/src/compile/snapshots/cubesql__compile__tests__formatted_null_string_is_empty.snap @@ -0,0 +1,9 @@ +--- +source: cubesql/src/compile/mod.rs +expression: result +--- ++--------------------------------+ +| formatted_null_string_is_empty | ++--------------------------------+ +| true | ++--------------------------------+ diff --git a/rust/cubesql/cubesql/src/compile/snapshots/cubesql__compile__tests__formatted_string.snap b/rust/cubesql/cubesql/src/compile/snapshots/cubesql__compile__tests__formatted_string.snap new file mode 100644 index 0000000000000..88e48153a0574 --- /dev/null +++ b/rust/cubesql/cubesql/src/compile/snapshots/cubesql__compile__tests__formatted_string.snap @@ -0,0 +1,9 @@ +--- +source: cubesql/src/compile/mod.rs +expression: result +--- ++------------------+ +| formatted_string | ++------------------+ +| foo | ++------------------+ diff --git a/rust/cubesql/cubesql/src/compile/snapshots/cubesql__compile__tests__formatted_strings.snap b/rust/cubesql/cubesql/src/compile/snapshots/cubesql__compile__tests__formatted_strings.snap new file mode 100644 index 0000000000000..3f0d49796427d --- /dev/null +++ b/rust/cubesql/cubesql/src/compile/snapshots/cubesql__compile__tests__formatted_strings.snap @@ -0,0 +1,9 @@ +--- +source: cubesql/src/compile/mod.rs +expression: result +--- ++-------------------+ +| formatted_strings | ++-------------------+ +| foo.bar | ++-------------------+ diff --git a/rust/cubesql/cubesql/src/compile/test/snapshots/cubesql__compile__test__test_introspection__test_metabase_v0_51_8_introspection.snap b/rust/cubesql/cubesql/src/compile/test/snapshots/cubesql__compile__test__test_introspection__test_metabase_v0_51_8_introspection.snap new file mode 100644 index 0000000000000..166491e9c523e --- /dev/null +++ b/rust/cubesql/cubesql/src/compile/test/snapshots/cubesql__compile__test__test_introspection__test_metabase_v0_51_8_introspection.snap @@ -0,0 +1,308 @@ +--- +source: cubesql/src/compile/test/test_introspection.rs +expression: "execute_query(r#\"\nselect\n \"c\".\"column_name\" as \"name\",\n case\n when \"c\".\"udt_schema\" in ('public', 'pg_catalog')\n then format('%s', \"c\".\"udt_name\")\n else format('\"%s\".\"%s\"', \"c\".\"udt_schema\", \"c\".\"udt_name\")\n end as \"database-type\",\n \"c\".\"ordinal_position\" - 1 as \"database-position\",\n \"c\".\"table_schema\" as \"table-schema\",\n \"c\".\"table_name\" as \"table-name\",\n \"pk\".\"column_name\" is not null as \"pk?\",\n col_description(\n cast(\n cast(\n format(\n '%I.%I',\n cast(\"c\".\"table_schema\" as text),\n cast(\"c\".\"table_name\" as text)\n ) as regclass\n ) as oid\n ),\n \"c\".\"ordinal_position\"\n ) as \"field-comment\",\n ((\"column_default\" is null) or (lower(\"column_default\") = 'null'))\n and (\"is_nullable\" = 'NO')\n and not (\n ((\"column_default\" is not null) and (\"column_default\" like '%nextval(%'))\n or (\"is_identity\" <> 'NO')\n ) as \"database-required\",\n ((\"column_default\" is not null) and (\"column_default\" like '%nextval(%'))\n or (\"is_identity\" <> 'NO') as \"database-is-auto-increment\"\nfrom \"information_schema\".\"columns\" as \"c\"\nleft join\n (\n select \"tc\".\"table_schema\", \"tc\".\"table_name\", \"kc\".\"column_name\"\n from \"information_schema\".\"table_constraints\" as \"tc\"\n inner join\n \"information_schema\".\"key_column_usage\" as \"kc\"\n on (\"tc\".\"constraint_name\" = \"kc\".\"constraint_name\")\n and (\"tc\".\"table_schema\" = \"kc\".\"table_schema\")\n and (\"tc\".\"table_name\" = \"kc\".\"table_name\")\n where \"tc\".\"constraint_type\" = 'PRIMARY KEY'\n ) as \"pk\"\n on (\"c\".\"table_schema\" = \"pk\".\"table_schema\")\n and (\"c\".\"table_name\" = \"pk\".\"table_name\")\n and (\"c\".\"column_name\" = \"pk\".\"column_name\")\nwhere\n c.table_schema !~ '^information_schema|catalog_history|pg_'\n and (\"c\".\"table_schema\" in ('public'))\nunion all\nselect\n \"pa\".\"attname\" as \"name\",\n case\n when \"ptn\".\"nspname\" in ('public', 'pg_catalog')\n then format('%s', \"pt\".\"typname\")\n else format('\"%s\".\"%s\"', \"ptn\".\"nspname\", \"pt\".\"typname\")\n end as \"database-type\",\n \"pa\".\"attnum\" - 1 as \"database-position\",\n \"pn\".\"nspname\" as \"table-schema\",\n \"pc\".\"relname\" as \"table-name\",\n false as \"pk?\",\n null as \"field-comment\",\n false as \"database-required\",\n false as \"database-is-auto-increment\"\nfrom \"pg_catalog\".\"pg_class\" as \"pc\"\ninner join \"pg_catalog\".\"pg_namespace\" as \"pn\" on \"pn\".\"oid\" = \"pc\".\"relnamespace\"\ninner join \"pg_catalog\".\"pg_attribute\" as \"pa\" on \"pa\".\"attrelid\" = \"pc\".\"oid\"\ninner join \"pg_catalog\".\"pg_type\" as \"pt\" on \"pt\".\"oid\" = \"pa\".\"atttypid\"\ninner join \"pg_catalog\".\"pg_namespace\" as \"ptn\" on \"ptn\".\"oid\" = \"pt\".\"typnamespace\"\nwhere (\"pc\".\"relkind\" = 'm') and (\"pa\".\"attnum\" >= 1) and (\"pn\".\"nspname\" in ('public'))\norder by \"table-schema\" asc, \"table-name\" asc, \"database-position\" asc\n \"#.to_string(),\nDatabaseProtocol::PostgreSQL).await?" +--- ++--------------------+---------------+-------------------+--------------+---------------------------+-------+---------------+-------------------+----------------------------+ +| name | database-type | database-position | table-schema | table-name | pk? | field-comment | database-required | database-is-auto-increment | ++--------------------+---------------+-------------------+--------------+---------------------------+-------+---------------+-------------------+----------------------------+ +| count | int8 | 0 | public | KibanaSampleDataEcommerce | false | NULL | false | false | +| maxPrice | numeric | 1 | public | KibanaSampleDataEcommerce | false | NULL | false | false | +| sumPrice | numeric | 2 | public | KibanaSampleDataEcommerce | false | NULL | false | false | +| minPrice | numeric | 3 | public | KibanaSampleDataEcommerce | false | NULL | false | false | +| avgPrice | numeric | 4 | public | KibanaSampleDataEcommerce | false | NULL | false | false | +| countDistinct | int8 | 5 | public | KibanaSampleDataEcommerce | false | NULL | false | false | +| order_date | timestamp | 6 | public | KibanaSampleDataEcommerce | false | NULL | false | false | +| last_mod | timestamp | 7 | public | KibanaSampleDataEcommerce | false | NULL | false | false | +| customer_gender | text | 8 | public | KibanaSampleDataEcommerce | false | NULL | false | false | +| notes | text | 9 | public | KibanaSampleDataEcommerce | false | NULL | false | false | +| taxful_total_price | numeric | 10 | public | KibanaSampleDataEcommerce | false | NULL | false | false | +| has_subscription | bool | 11 | public | KibanaSampleDataEcommerce | false | NULL | false | false | +| is_male | bool | 12 | public | KibanaSampleDataEcommerce | false | NULL | false | false | +| is_female | bool | 13 | public | KibanaSampleDataEcommerce | false | NULL | false | false | +| __user | text | 14 | public | KibanaSampleDataEcommerce | false | NULL | false | false | +| __cubeJoinField | text | 15 | public | KibanaSampleDataEcommerce | false | NULL | false | false | +| agentCount | int8 | 0 | public | Logs | false | NULL | false | false | +| agentCountApprox | int8 | 1 | public | Logs | false | NULL | false | false | +| id | numeric | 2 | public | Logs | false | NULL | false | false | +| read | bool | 3 | public | Logs | false | NULL | false | false | +| content | text | 4 | public | Logs | false | NULL | false | false | +| __user | text | 5 | public | Logs | false | NULL | false | false | +| __cubeJoinField | text | 6 | public | Logs | false | NULL | false | false | +| measure_num0 | numeric | 0 | public | MultiTypeCube | false | NULL | false | false | +| measure_str0 | numeric | 1 | public | MultiTypeCube | false | NULL | false | false | +| measure_date0 | numeric | 2 | public | MultiTypeCube | false | NULL | false | false | +| measure_num1 | numeric | 3 | public | MultiTypeCube | false | NULL | false | false | +| measure_str1 | numeric | 4 | public | MultiTypeCube | false | NULL | false | false | +| measure_date1 | numeric | 5 | public | MultiTypeCube | false | NULL | false | false | +| measure_num2 | numeric | 6 | public | MultiTypeCube | false | NULL | false | false | +| measure_str2 | numeric | 7 | public | MultiTypeCube | false | NULL | false | false | +| measure_date2 | numeric | 8 | public | MultiTypeCube | false | NULL | false | false | +| measure_num3 | numeric | 9 | public | MultiTypeCube | false | NULL | false | false | +| measure_str3 | numeric | 10 | public | MultiTypeCube | false | NULL | false | false | +| measure_date3 | numeric | 11 | public | MultiTypeCube | false | NULL | false | false | +| measure_num4 | numeric | 12 | public | MultiTypeCube | false | NULL | false | false | +| measure_str4 | numeric | 13 | public | MultiTypeCube | false | NULL | false | false | +| measure_date4 | numeric | 14 | public | MultiTypeCube | false | NULL | false | false | +| measure_num5 | numeric | 15 | public | MultiTypeCube | false | NULL | false | false | +| measure_str5 | numeric | 16 | public | MultiTypeCube | false | NULL | false | false | +| measure_date5 | numeric | 17 | public | MultiTypeCube | false | NULL | false | false | +| measure_num6 | numeric | 18 | public | MultiTypeCube | false | NULL | false | false | +| measure_str6 | numeric | 19 | public | MultiTypeCube | false | NULL | false | false | +| measure_date6 | numeric | 20 | public | MultiTypeCube | false | NULL | false | false | +| measure_num7 | numeric | 21 | public | MultiTypeCube | false | NULL | false | false | +| measure_str7 | numeric | 22 | public | MultiTypeCube | false | NULL | false | false | +| measure_date7 | numeric | 23 | public | MultiTypeCube | false | NULL | false | false | +| measure_num8 | numeric | 24 | public | MultiTypeCube | false | NULL | false | false | +| measure_str8 | numeric | 25 | public | MultiTypeCube | false | NULL | false | false | +| measure_date8 | numeric | 26 | public | MultiTypeCube | false | NULL | false | false | +| measure_num9 | numeric | 27 | public | MultiTypeCube | false | NULL | false | false | +| measure_str9 | numeric | 28 | public | MultiTypeCube | false | NULL | false | false | +| measure_date9 | numeric | 29 | public | MultiTypeCube | false | NULL | false | false | +| count | int8 | 30 | public | MultiTypeCube | false | NULL | false | false | +| maxPrice | numeric | 31 | public | MultiTypeCube | false | NULL | false | false | +| minPrice | numeric | 32 | public | MultiTypeCube | false | NULL | false | false | +| avgPrice | numeric | 33 | public | MultiTypeCube | false | NULL | false | false | +| countDistinct | int8 | 34 | public | MultiTypeCube | false | NULL | false | false | +| dim_num0 | numeric | 35 | public | MultiTypeCube | false | NULL | false | false | +| dim_str0 | text | 36 | public | MultiTypeCube | false | NULL | false | false | +| dim_date0 | timestamp | 37 | public | MultiTypeCube | false | NULL | false | false | +| dim_num1 | numeric | 38 | public | MultiTypeCube | false | NULL | false | false | +| dim_str1 | text | 39 | public | MultiTypeCube | false | NULL | false | false | +| dim_date1 | timestamp | 40 | public | MultiTypeCube | false | NULL | false | false | +| dim_num2 | numeric | 41 | public | MultiTypeCube | false | NULL | false | false | +| dim_str2 | text | 42 | public | MultiTypeCube | false | NULL | false | false | +| dim_date2 | timestamp | 43 | public | MultiTypeCube | false | NULL | false | false | +| dim_num3 | numeric | 44 | public | MultiTypeCube | false | NULL | false | false | +| dim_str3 | text | 45 | public | MultiTypeCube | false | NULL | false | false | +| dim_date3 | timestamp | 46 | public | MultiTypeCube | false | NULL | false | false | +| dim_num4 | numeric | 47 | public | MultiTypeCube | false | NULL | false | false | +| dim_str4 | text | 48 | public | MultiTypeCube | false | NULL | false | false | +| dim_date4 | timestamp | 49 | public | MultiTypeCube | false | NULL | false | false | +| dim_num5 | numeric | 50 | public | MultiTypeCube | false | NULL | false | false | +| dim_str5 | text | 51 | public | MultiTypeCube | false | NULL | false | false | +| dim_date5 | timestamp | 52 | public | MultiTypeCube | false | NULL | false | false | +| dim_num6 | numeric | 53 | public | MultiTypeCube | false | NULL | false | false | +| dim_str6 | text | 54 | public | MultiTypeCube | false | NULL | false | false | +| dim_date6 | timestamp | 55 | public | MultiTypeCube | false | NULL | false | false | +| dim_num7 | numeric | 56 | public | MultiTypeCube | false | NULL | false | false | +| dim_str7 | text | 57 | public | MultiTypeCube | false | NULL | false | false | +| dim_date7 | timestamp | 58 | public | MultiTypeCube | false | NULL | false | false | +| dim_num8 | numeric | 59 | public | MultiTypeCube | false | NULL | false | false | +| dim_str8 | text | 60 | public | MultiTypeCube | false | NULL | false | false | +| dim_date8 | timestamp | 61 | public | MultiTypeCube | false | NULL | false | false | +| dim_num9 | numeric | 62 | public | MultiTypeCube | false | NULL | false | false | +| dim_str9 | text | 63 | public | MultiTypeCube | false | NULL | false | false | +| dim_date9 | timestamp | 64 | public | MultiTypeCube | false | NULL | false | false | +| __user | text | 65 | public | MultiTypeCube | false | NULL | false | false | +| __cubeJoinField | text | 66 | public | MultiTypeCube | false | NULL | false | false | +| someNumber | numeric | 0 | public | NumberCube | false | NULL | false | false | +| __user | text | 1 | public | NumberCube | false | NULL | false | false | +| __cubeJoinField | text | 2 | public | NumberCube | false | NULL | false | false | +| measure0 | numeric | 0 | public | WideCube | false | NULL | false | false | +| measure1 | numeric | 1 | public | WideCube | false | NULL | false | false | +| measure2 | numeric | 2 | public | WideCube | false | NULL | false | false | +| measure3 | numeric | 3 | public | WideCube | false | NULL | false | false | +| measure4 | numeric | 4 | public | WideCube | false | NULL | false | false | +| measure5 | numeric | 5 | public | WideCube | false | NULL | false | false | +| measure6 | numeric | 6 | public | WideCube | false | NULL | false | false | +| measure7 | numeric | 7 | public | WideCube | false | NULL | false | false | +| measure8 | numeric | 8 | public | WideCube | false | NULL | false | false | +| measure9 | numeric | 9 | public | WideCube | false | NULL | false | false | +| measure10 | numeric | 10 | public | WideCube | false | NULL | false | false | +| measure11 | numeric | 11 | public | WideCube | false | NULL | false | false | +| measure12 | numeric | 12 | public | WideCube | false | NULL | false | false | +| measure13 | numeric | 13 | public | WideCube | false | NULL | false | false | +| measure14 | numeric | 14 | public | WideCube | false | NULL | false | false | +| measure15 | numeric | 15 | public | WideCube | false | NULL | false | false | +| measure16 | numeric | 16 | public | WideCube | false | NULL | false | false | +| measure17 | numeric | 17 | public | WideCube | false | NULL | false | false | +| measure18 | numeric | 18 | public | WideCube | false | NULL | false | false | +| measure19 | numeric | 19 | public | WideCube | false | NULL | false | false | +| measure20 | numeric | 20 | public | WideCube | false | NULL | false | false | +| measure21 | numeric | 21 | public | WideCube | false | NULL | false | false | +| measure22 | numeric | 22 | public | WideCube | false | NULL | false | false | +| measure23 | numeric | 23 | public | WideCube | false | NULL | false | false | +| measure24 | numeric | 24 | public | WideCube | false | NULL | false | false | +| measure25 | numeric | 25 | public | WideCube | false | NULL | false | false | +| measure26 | numeric | 26 | public | WideCube | false | NULL | false | false | +| measure27 | numeric | 27 | public | WideCube | false | NULL | false | false | +| measure28 | numeric | 28 | public | WideCube | false | NULL | false | false | +| measure29 | numeric | 29 | public | WideCube | false | NULL | false | false | +| measure30 | numeric | 30 | public | WideCube | false | NULL | false | false | +| measure31 | numeric | 31 | public | WideCube | false | NULL | false | false | +| measure32 | numeric | 32 | public | WideCube | false | NULL | false | false | +| measure33 | numeric | 33 | public | WideCube | false | NULL | false | false | +| measure34 | numeric | 34 | public | WideCube | false | NULL | false | false | +| measure35 | numeric | 35 | public | WideCube | false | NULL | false | false | +| measure36 | numeric | 36 | public | WideCube | false | NULL | false | false | +| measure37 | numeric | 37 | public | WideCube | false | NULL | false | false | +| measure38 | numeric | 38 | public | WideCube | false | NULL | false | false | +| measure39 | numeric | 39 | public | WideCube | false | NULL | false | false | +| measure40 | numeric | 40 | public | WideCube | false | NULL | false | false | +| measure41 | numeric | 41 | public | WideCube | false | NULL | false | false | +| measure42 | numeric | 42 | public | WideCube | false | NULL | false | false | +| measure43 | numeric | 43 | public | WideCube | false | NULL | false | false | +| measure44 | numeric | 44 | public | WideCube | false | NULL | false | false | +| measure45 | numeric | 45 | public | WideCube | false | NULL | false | false | +| measure46 | numeric | 46 | public | WideCube | false | NULL | false | false | +| measure47 | numeric | 47 | public | WideCube | false | NULL | false | false | +| measure48 | numeric | 48 | public | WideCube | false | NULL | false | false | +| measure49 | numeric | 49 | public | WideCube | false | NULL | false | false | +| measure50 | numeric | 50 | public | WideCube | false | NULL | false | false | +| measure51 | numeric | 51 | public | WideCube | false | NULL | false | false | +| measure52 | numeric | 52 | public | WideCube | false | NULL | false | false | +| measure53 | numeric | 53 | public | WideCube | false | NULL | false | false | +| measure54 | numeric | 54 | public | WideCube | false | NULL | false | false | +| measure55 | numeric | 55 | public | WideCube | false | NULL | false | false | +| measure56 | numeric | 56 | public | WideCube | false | NULL | false | false | +| measure57 | numeric | 57 | public | WideCube | false | NULL | false | false | +| measure58 | numeric | 58 | public | WideCube | false | NULL | false | false | +| measure59 | numeric | 59 | public | WideCube | false | NULL | false | false | +| measure60 | numeric | 60 | public | WideCube | false | NULL | false | false | +| measure61 | numeric | 61 | public | WideCube | false | NULL | false | false | +| measure62 | numeric | 62 | public | WideCube | false | NULL | false | false | +| measure63 | numeric | 63 | public | WideCube | false | NULL | false | false | +| measure64 | numeric | 64 | public | WideCube | false | NULL | false | false | +| measure65 | numeric | 65 | public | WideCube | false | NULL | false | false | +| measure66 | numeric | 66 | public | WideCube | false | NULL | false | false | +| measure67 | numeric | 67 | public | WideCube | false | NULL | false | false | +| measure68 | numeric | 68 | public | WideCube | false | NULL | false | false | +| measure69 | numeric | 69 | public | WideCube | false | NULL | false | false | +| measure70 | numeric | 70 | public | WideCube | false | NULL | false | false | +| measure71 | numeric | 71 | public | WideCube | false | NULL | false | false | +| measure72 | numeric | 72 | public | WideCube | false | NULL | false | false | +| measure73 | numeric | 73 | public | WideCube | false | NULL | false | false | +| measure74 | numeric | 74 | public | WideCube | false | NULL | false | false | +| measure75 | numeric | 75 | public | WideCube | false | NULL | false | false | +| measure76 | numeric | 76 | public | WideCube | false | NULL | false | false | +| measure77 | numeric | 77 | public | WideCube | false | NULL | false | false | +| measure78 | numeric | 78 | public | WideCube | false | NULL | false | false | +| measure79 | numeric | 79 | public | WideCube | false | NULL | false | false | +| measure80 | numeric | 80 | public | WideCube | false | NULL | false | false | +| measure81 | numeric | 81 | public | WideCube | false | NULL | false | false | +| measure82 | numeric | 82 | public | WideCube | false | NULL | false | false | +| measure83 | numeric | 83 | public | WideCube | false | NULL | false | false | +| measure84 | numeric | 84 | public | WideCube | false | NULL | false | false | +| measure85 | numeric | 85 | public | WideCube | false | NULL | false | false | +| measure86 | numeric | 86 | public | WideCube | false | NULL | false | false | +| measure87 | numeric | 87 | public | WideCube | false | NULL | false | false | +| measure88 | numeric | 88 | public | WideCube | false | NULL | false | false | +| measure89 | numeric | 89 | public | WideCube | false | NULL | false | false | +| measure90 | numeric | 90 | public | WideCube | false | NULL | false | false | +| measure91 | numeric | 91 | public | WideCube | false | NULL | false | false | +| measure92 | numeric | 92 | public | WideCube | false | NULL | false | false | +| measure93 | numeric | 93 | public | WideCube | false | NULL | false | false | +| measure94 | numeric | 94 | public | WideCube | false | NULL | false | false | +| measure95 | numeric | 95 | public | WideCube | false | NULL | false | false | +| measure96 | numeric | 96 | public | WideCube | false | NULL | false | false | +| measure97 | numeric | 97 | public | WideCube | false | NULL | false | false | +| measure98 | numeric | 98 | public | WideCube | false | NULL | false | false | +| measure99 | numeric | 99 | public | WideCube | false | NULL | false | false | +| count | int8 | 100 | public | WideCube | false | NULL | false | false | +| maxPrice | numeric | 101 | public | WideCube | false | NULL | false | false | +| minPrice | numeric | 102 | public | WideCube | false | NULL | false | false | +| avgPrice | numeric | 103 | public | WideCube | false | NULL | false | false | +| countDistinct | int8 | 104 | public | WideCube | false | NULL | false | false | +| dim0 | numeric | 105 | public | WideCube | false | NULL | false | false | +| dim1 | numeric | 106 | public | WideCube | false | NULL | false | false | +| dim2 | numeric | 107 | public | WideCube | false | NULL | false | false | +| dim3 | numeric | 108 | public | WideCube | false | NULL | false | false | +| dim4 | numeric | 109 | public | WideCube | false | NULL | false | false | +| dim5 | numeric | 110 | public | WideCube | false | NULL | false | false | +| dim6 | numeric | 111 | public | WideCube | false | NULL | false | false | +| dim7 | numeric | 112 | public | WideCube | false | NULL | false | false | +| dim8 | numeric | 113 | public | WideCube | false | NULL | false | false | +| dim9 | numeric | 114 | public | WideCube | false | NULL | false | false | +| dim10 | numeric | 115 | public | WideCube | false | NULL | false | false | +| dim11 | numeric | 116 | public | WideCube | false | NULL | false | false | +| dim12 | numeric | 117 | public | WideCube | false | NULL | false | false | +| dim13 | numeric | 118 | public | WideCube | false | NULL | false | false | +| dim14 | numeric | 119 | public | WideCube | false | NULL | false | false | +| dim15 | numeric | 120 | public | WideCube | false | NULL | false | false | +| dim16 | numeric | 121 | public | WideCube | false | NULL | false | false | +| dim17 | numeric | 122 | public | WideCube | false | NULL | false | false | +| dim18 | numeric | 123 | public | WideCube | false | NULL | false | false | +| dim19 | numeric | 124 | public | WideCube | false | NULL | false | false | +| dim20 | numeric | 125 | public | WideCube | false | NULL | false | false | +| dim21 | numeric | 126 | public | WideCube | false | NULL | false | false | +| dim22 | numeric | 127 | public | WideCube | false | NULL | false | false | +| dim23 | numeric | 128 | public | WideCube | false | NULL | false | false | +| dim24 | numeric | 129 | public | WideCube | false | NULL | false | false | +| dim25 | numeric | 130 | public | WideCube | false | NULL | false | false | +| dim26 | numeric | 131 | public | WideCube | false | NULL | false | false | +| dim27 | numeric | 132 | public | WideCube | false | NULL | false | false | +| dim28 | numeric | 133 | public | WideCube | false | NULL | false | false | +| dim29 | numeric | 134 | public | WideCube | false | NULL | false | false | +| dim30 | numeric | 135 | public | WideCube | false | NULL | false | false | +| dim31 | numeric | 136 | public | WideCube | false | NULL | false | false | +| dim32 | numeric | 137 | public | WideCube | false | NULL | false | false | +| dim33 | numeric | 138 | public | WideCube | false | NULL | false | false | +| dim34 | numeric | 139 | public | WideCube | false | NULL | false | false | +| dim35 | numeric | 140 | public | WideCube | false | NULL | false | false | +| dim36 | numeric | 141 | public | WideCube | false | NULL | false | false | +| dim37 | numeric | 142 | public | WideCube | false | NULL | false | false | +| dim38 | numeric | 143 | public | WideCube | false | NULL | false | false | +| dim39 | numeric | 144 | public | WideCube | false | NULL | false | false | +| dim40 | numeric | 145 | public | WideCube | false | NULL | false | false | +| dim41 | numeric | 146 | public | WideCube | false | NULL | false | false | +| dim42 | numeric | 147 | public | WideCube | false | NULL | false | false | +| dim43 | numeric | 148 | public | WideCube | false | NULL | false | false | +| dim44 | numeric | 149 | public | WideCube | false | NULL | false | false | +| dim45 | numeric | 150 | public | WideCube | false | NULL | false | false | +| dim46 | numeric | 151 | public | WideCube | false | NULL | false | false | +| dim47 | numeric | 152 | public | WideCube | false | NULL | false | false | +| dim48 | numeric | 153 | public | WideCube | false | NULL | false | false | +| dim49 | numeric | 154 | public | WideCube | false | NULL | false | false | +| dim50 | numeric | 155 | public | WideCube | false | NULL | false | false | +| dim51 | numeric | 156 | public | WideCube | false | NULL | false | false | +| dim52 | numeric | 157 | public | WideCube | false | NULL | false | false | +| dim53 | numeric | 158 | public | WideCube | false | NULL | false | false | +| dim54 | numeric | 159 | public | WideCube | false | NULL | false | false | +| dim55 | numeric | 160 | public | WideCube | false | NULL | false | false | +| dim56 | numeric | 161 | public | WideCube | false | NULL | false | false | +| dim57 | numeric | 162 | public | WideCube | false | NULL | false | false | +| dim58 | numeric | 163 | public | WideCube | false | NULL | false | false | +| dim59 | numeric | 164 | public | WideCube | false | NULL | false | false | +| dim60 | numeric | 165 | public | WideCube | false | NULL | false | false | +| dim61 | numeric | 166 | public | WideCube | false | NULL | false | false | +| dim62 | numeric | 167 | public | WideCube | false | NULL | false | false | +| dim63 | numeric | 168 | public | WideCube | false | NULL | false | false | +| dim64 | numeric | 169 | public | WideCube | false | NULL | false | false | +| dim65 | numeric | 170 | public | WideCube | false | NULL | false | false | +| dim66 | numeric | 171 | public | WideCube | false | NULL | false | false | +| dim67 | numeric | 172 | public | WideCube | false | NULL | false | false | +| dim68 | numeric | 173 | public | WideCube | false | NULL | false | false | +| dim69 | numeric | 174 | public | WideCube | false | NULL | false | false | +| dim70 | numeric | 175 | public | WideCube | false | NULL | false | false | +| dim71 | numeric | 176 | public | WideCube | false | NULL | false | false | +| dim72 | numeric | 177 | public | WideCube | false | NULL | false | false | +| dim73 | numeric | 178 | public | WideCube | false | NULL | false | false | +| dim74 | numeric | 179 | public | WideCube | false | NULL | false | false | +| dim75 | numeric | 180 | public | WideCube | false | NULL | false | false | +| dim76 | numeric | 181 | public | WideCube | false | NULL | false | false | +| dim77 | numeric | 182 | public | WideCube | false | NULL | false | false | +| dim78 | numeric | 183 | public | WideCube | false | NULL | false | false | +| dim79 | numeric | 184 | public | WideCube | false | NULL | false | false | +| dim80 | numeric | 185 | public | WideCube | false | NULL | false | false | +| dim81 | numeric | 186 | public | WideCube | false | NULL | false | false | +| dim82 | numeric | 187 | public | WideCube | false | NULL | false | false | +| dim83 | numeric | 188 | public | WideCube | false | NULL | false | false | +| dim84 | numeric | 189 | public | WideCube | false | NULL | false | false | +| dim85 | numeric | 190 | public | WideCube | false | NULL | false | false | +| dim86 | numeric | 191 | public | WideCube | false | NULL | false | false | +| dim87 | numeric | 192 | public | WideCube | false | NULL | false | false | +| dim88 | numeric | 193 | public | WideCube | false | NULL | false | false | +| dim89 | numeric | 194 | public | WideCube | false | NULL | false | false | +| dim90 | numeric | 195 | public | WideCube | false | NULL | false | false | +| dim91 | numeric | 196 | public | WideCube | false | NULL | false | false | +| dim92 | numeric | 197 | public | WideCube | false | NULL | false | false | +| dim93 | numeric | 198 | public | WideCube | false | NULL | false | false | +| dim94 | numeric | 199 | public | WideCube | false | NULL | false | false | +| dim95 | numeric | 200 | public | WideCube | false | NULL | false | false | +| dim96 | numeric | 201 | public | WideCube | false | NULL | false | false | +| dim97 | numeric | 202 | public | WideCube | false | NULL | false | false | +| dim98 | numeric | 203 | public | WideCube | false | NULL | false | false | +| dim99 | numeric | 204 | public | WideCube | false | NULL | false | false | +| __user | text | 205 | public | WideCube | false | NULL | false | false | +| __cubeJoinField | text | 206 | public | WideCube | false | NULL | false | false | ++--------------------+---------------+-------------------+--------------+---------------------------+-------+---------------+-------------------+----------------------------+ diff --git a/rust/cubesql/cubesql/src/compile/test/test_introspection.rs b/rust/cubesql/cubesql/src/compile/test/test_introspection.rs index e0e40723d2f4d..0a4f3b9fc0f24 100644 --- a/rust/cubesql/cubesql/src/compile/test/test_introspection.rs +++ b/rust/cubesql/cubesql/src/compile/test/test_introspection.rs @@ -3218,3 +3218,92 @@ async fn test_metabase_v0_51_2_introspection_field_indoption() -> Result<(), Cub ); Ok(()) } + +#[tokio::test] +async fn test_metabase_v0_51_8_introspection() -> Result<(), CubeError> { + init_testing_logger(); + + insta::assert_snapshot!( + "test_metabase_v0_51_8_introspection", + execute_query( + // language=PostgreSQL + r#" +select + "c"."column_name" as "name", + case + when "c"."udt_schema" in ('public', 'pg_catalog') + then format('%s', "c"."udt_name") + else format('"%s"."%s"', "c"."udt_schema", "c"."udt_name") + end as "database-type", + "c"."ordinal_position" - 1 as "database-position", + "c"."table_schema" as "table-schema", + "c"."table_name" as "table-name", + "pk"."column_name" is not null as "pk?", + col_description( + cast( + cast( + format( + '%I.%I', + cast("c"."table_schema" as text), + cast("c"."table_name" as text) + ) as regclass + ) as oid + ), + "c"."ordinal_position" + ) as "field-comment", + (("column_default" is null) or (lower("column_default") = 'null')) + and ("is_nullable" = 'NO') + and not ( + (("column_default" is not null) and ("column_default" like '%nextval(%')) + or ("is_identity" <> 'NO') + ) as "database-required", + (("column_default" is not null) and ("column_default" like '%nextval(%')) + or ("is_identity" <> 'NO') as "database-is-auto-increment" +from "information_schema"."columns" as "c" +left join + ( + select "tc"."table_schema", "tc"."table_name", "kc"."column_name" + from "information_schema"."table_constraints" as "tc" + inner join + "information_schema"."key_column_usage" as "kc" + on ("tc"."constraint_name" = "kc"."constraint_name") + and ("tc"."table_schema" = "kc"."table_schema") + and ("tc"."table_name" = "kc"."table_name") + where "tc"."constraint_type" = 'PRIMARY KEY' + ) as "pk" + on ("c"."table_schema" = "pk"."table_schema") + and ("c"."table_name" = "pk"."table_name") + and ("c"."column_name" = "pk"."column_name") +where + c.table_schema !~ '^information_schema|catalog_history|pg_' + and ("c"."table_schema" in ('public')) +union all +select + "pa"."attname" as "name", + case + when "ptn"."nspname" in ('public', 'pg_catalog') + then format('%s', "pt"."typname") + else format('"%s"."%s"', "ptn"."nspname", "pt"."typname") + end as "database-type", + "pa"."attnum" - 1 as "database-position", + "pn"."nspname" as "table-schema", + "pc"."relname" as "table-name", + false as "pk?", + null as "field-comment", + false as "database-required", + false as "database-is-auto-increment" +from "pg_catalog"."pg_class" as "pc" +inner join "pg_catalog"."pg_namespace" as "pn" on "pn"."oid" = "pc"."relnamespace" +inner join "pg_catalog"."pg_attribute" as "pa" on "pa"."attrelid" = "pc"."oid" +inner join "pg_catalog"."pg_type" as "pt" on "pt"."oid" = "pa"."atttypid" +inner join "pg_catalog"."pg_namespace" as "ptn" on "ptn"."oid" = "pt"."typnamespace" +where ("pc"."relkind" = 'm') and ("pa"."attnum" >= 1) and ("pn"."nspname" in ('public')) +order by "table-schema" asc, "table-name" asc, "database-position" asc + "# + .to_string(), + DatabaseProtocol::PostgreSQL + ) + .await? + ); + Ok(()) +} From e7fd576f55a7b0a129c44e05a415017c50cdc24f Mon Sep 17 00:00:00 2001 From: Pavel Tiunov Date: Thu, 23 Jan 2025 14:54:53 -0800 Subject: [PATCH 2/5] docs: Elaborate on stability of context to app id for multi-cluster --- docs/pages/product/deployment/cloud/deployment-types.mdx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/pages/product/deployment/cloud/deployment-types.mdx b/docs/pages/product/deployment/cloud/deployment-types.mdx index 4c388cc94841c..bb2592783a16a 100644 --- a/docs/pages/product/deployment/cloud/deployment-types.mdx +++ b/docs/pages/product/deployment/cloud/deployment-types.mdx @@ -165,6 +165,7 @@ module.exports = { If your implementation of `context_to_app_id` returns identifiers that change over time for each tenant, requests from one tenant would likely hit multiple production clusters and you would not have the benefit of reduced memory footprint. +Also you might see 502 or timeout errors in case of different cluster nodes would return different `context_to_app_id` results for the same request. @@ -183,4 +184,4 @@ and select from the available options: [ref-scalability]: /product/deployment/cloud/scalability [ref-multitenancy]: /product/configuration/advanced/multitenancy [ref-auto-sus]: /product/deployment/cloud/auto-suspension -[ref-refresh-worker]: /product/deployment#refresh-worker \ No newline at end of file +[ref-refresh-worker]: /product/deployment#refresh-worker From 5a540db9228dcbb88c434123f13291202f6da9be Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 24 Jan 2025 11:31:33 +0200 Subject: [PATCH 3/5] =?UTF-8?q?feat(schema-compiler):=20Add=20flag=20for?= =?UTF-8?q?=20using=20named=20timezones=20in=C2=A0MySQL=20Query=20class=20?= =?UTF-8?q?(#9111)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Remove moment-timezone from vertica driver * Remove moment-timezone from druid driver * Remove moment-timezone from dremio driver * Remove moment-timezone from cubestore driver * update moment-timezone in api-gateway * update moment-timezone in backend-shared * Remove moment-timezone from query-orchestrator * update moment-timezone in schema-compiler * remove unused * remove moment.HTML5_FMT.DATETIME_LOCAL_MS in favor of just literal * linter fix * add mysqlUseNamedTimezones flag * fix dremio test * Tiny edits --------- Co-authored-by: Igor Lukanin --- .../configuration/data-sources/mysql.mdx | 1 + .../configuration/environment-variables.mdx | 12 ++++++ packages/cubejs-api-gateway/package.json | 2 +- packages/cubejs-backend-shared/package.json | 2 +- packages/cubejs-backend-shared/src/env.ts | 38 +++++++++++++++++++ packages/cubejs-cubestore-driver/package.json | 1 - .../driver/DremioQuery.js | 13 +++++-- packages/cubejs-dremio-driver/package.json | 1 - .../test/DremioQuery.test.ts | 2 +- packages/cubejs-druid-driver/package.json | 3 +- .../cubejs-druid-driver/src/DruidQuery.ts | 1 - .../cubejs-query-orchestrator/package.json | 1 - packages/cubejs-schema-compiler/package.json | 2 +- .../src/adapter/BaseQuery.js | 4 -- .../src/adapter/CubeStoreQuery.ts | 2 +- .../src/adapter/MysqlQuery.ts | 31 +++++++++------ packages/cubejs-vertica-driver/package.json | 1 - .../cubejs-vertica-driver/src/VerticaQuery.js | 3 +- yarn.lock | 6 +-- 19 files changed, 90 insertions(+), 36 deletions(-) diff --git a/docs/pages/product/configuration/data-sources/mysql.mdx b/docs/pages/product/configuration/data-sources/mysql.mdx index cbee48d92c451..3d0b1c21a5630 100644 --- a/docs/pages/product/configuration/data-sources/mysql.mdx +++ b/docs/pages/product/configuration/data-sources/mysql.mdx @@ -38,6 +38,7 @@ CUBEJS_DB_PASS=********** | `CUBEJS_DB_SSL` | If `true`, enables SSL encryption for database connections from Cube | `true`, `false` | ❌ | | `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number | ❌ | | `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8` | A valid number | ❌ | +| `CUBEJS_DB_MYSQL_USE_NAMED_TIMEZONES` | The flag to use time zone names or numeric offsets for time zone conversion. Default is `false` | `true`, `false` | ❌ | ## Pre-Aggregation Feature Support diff --git a/docs/pages/reference/configuration/environment-variables.mdx b/docs/pages/reference/configuration/environment-variables.mdx index 1b7ac21aacae2..f227eb192cba5 100644 --- a/docs/pages/reference/configuration/environment-variables.mdx +++ b/docs/pages/reference/configuration/environment-variables.mdx @@ -663,6 +663,17 @@ The cluster name to use when connecting to [Materialize](/product/configuration/ | --------------------------------------------------------- | ---------------------- | --------------------- | | A valid Materialize cluster name | N/A | N/A | +## `CUBEJS_DB_MYSQL_USE_NAMED_TIMEZONES` + +This flag controls how time zones conversion is done in the generated SQL for MySQL: +- If it is set to `true`, time zone names are used. In this case, your MySQL server needs +to be [configured][mysql-server-tz-support] properly. +- If it is set to `false`, numeric offsets are used instead. + +| Possible Values | Default in Development | Default in Production | +| --------------- | ---------------------- | --------------------- | +| `true`, `false` | `false` | `false` | + ## `CUBEJS_DB_SNOWFLAKE_ACCOUNT` The Snowflake account identifier to use when connecting to the database. @@ -1612,3 +1623,4 @@ The port for a Cube deployment to listen to API connections on. [ref-sql-api]: /product/apis-integrations/sql-api [ref-sql-api-streaming]: /product/apis-integrations/sql-api#streaming [ref-row-limit]: /product/apis-integrations/queries#row-limit +[mysql-server-tz-support]: https://dev.mysql.com/doc/refman/8.4/en/time-zone-support.html diff --git a/packages/cubejs-api-gateway/package.json b/packages/cubejs-api-gateway/package.json index a26097cd0dd5f..22256b6e0f750 100644 --- a/packages/cubejs-api-gateway/package.json +++ b/packages/cubejs-api-gateway/package.json @@ -43,7 +43,7 @@ "jsonwebtoken": "^9.0.2", "jwk-to-pem": "^2.0.4", "moment": "^2.24.0", - "moment-timezone": "^0.5.27", + "moment-timezone": "^0.5.46", "nexus": "^1.1.0", "node-fetch": "^2.6.1", "ramda": "^0.27.0", diff --git a/packages/cubejs-backend-shared/package.json b/packages/cubejs-backend-shared/package.json index d0a59c6b98f42..691caf3fdee2e 100644 --- a/packages/cubejs-backend-shared/package.json +++ b/packages/cubejs-backend-shared/package.json @@ -44,7 +44,7 @@ "fs-extra": "^9.1.0", "http-proxy-agent": "^4.0.1", "moment-range": "^4.0.1", - "moment-timezone": "^0.5.33", + "moment-timezone": "^0.5.46", "node-fetch": "^2.6.1", "shelljs": "^0.8.5", "throttle-debounce": "^3.0.1", diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index 36f31eb5f58a1..36a4bf4e7c41b 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -866,6 +866,44 @@ const variables: Record any> = { return undefined; }, + /** **************************************************************** + * MySQL Driver * + ***************************************************************** */ + + /** + * Use timezone names for date/time conversions. + * Defaults to FALSE, meaning that numeric offsets for timezone will be used. + * @see https://dev.mysql.com/doc/refman/8.4/en/date-and-time-functions.html#function_convert-tz + * @see https://dev.mysql.com/doc/refman/8.4/en/time-zone-support.html + */ + mysqlUseNamedTimezones: ({ dataSource }: { dataSource: string }) => { + const val = process.env[ + keyByDataSource( + 'CUBEJS_DB_MYSQL_USE_NAMED_TIMEZONES', + dataSource, + ) + ]; + + if (val) { + if (val.toLocaleLowerCase() === 'true') { + return true; + } else if (val.toLowerCase() === 'false') { + return false; + } else { + throw new TypeError( + `The ${ + keyByDataSource( + 'CUBEJS_DB_MYSQL_USE_NAMED_TIMEZONES', + dataSource, + ) + } must be either 'true' or 'false'.` + ); + } + } else { + return false; + } + }, + /** **************************************************************** * Databricks Driver * ***************************************************************** */ diff --git a/packages/cubejs-cubestore-driver/package.json b/packages/cubejs-cubestore-driver/package.json index 10f04e2923174..125e69e55acbf 100644 --- a/packages/cubejs-cubestore-driver/package.json +++ b/packages/cubejs-cubestore-driver/package.json @@ -33,7 +33,6 @@ "flatbuffers": "23.3.3", "fs-extra": "^9.1.0", "generic-pool": "^3.6.0", - "moment-timezone": "^0.5.31", "node-fetch": "^2.6.1", "sqlstring": "^2.3.3", "tempy": "^1.0.1", diff --git a/packages/cubejs-dremio-driver/driver/DremioQuery.js b/packages/cubejs-dremio-driver/driver/DremioQuery.js index 94c00e07be82b..c46ade9c517df 100644 --- a/packages/cubejs-dremio-driver/driver/DremioQuery.js +++ b/packages/cubejs-dremio-driver/driver/DremioQuery.js @@ -1,4 +1,3 @@ -const moment = require('moment-timezone'); const { BaseFilter, BaseQuery } = require('@cubejs-backend/schema-compiler'); const GRANULARITY_TO_INTERVAL = { @@ -36,9 +35,15 @@ class DremioQuery extends BaseQuery { return new DremioFilter(this, filter); } + /** + * CONVERT_TIMEZONE([sourceTimezone string], destinationTimezone string, + * timestamp date, timestamp, or string in ISO 8601 format) → timestamp + * sourceTimezone (optional): The time zone of the timestamp. If you omit this parameter, + * Dremio assumes that the source time zone is UTC. + * @see https://docs.dremio.com/cloud/reference/sql/sql-functions/functions/CONVERT_TIMEZONE/ + */ convertTz(field) { - const targetTZ = moment().tz(this.timezone).format('Z'); - return `CONVERT_TIMEZONE('${targetTZ}', ${field})`; + return `CONVERT_TIMEZONE('${this.timezone}', ${field})`; } timeStampCast(value) { @@ -46,7 +51,7 @@ class DremioQuery extends BaseQuery { } timestampFormat() { - return moment.HTML5_FMT.DATETIME_LOCAL_MS; + return 'YYYY-MM-DDTHH:mm:ss.SSS'; } dateTimeCast(value) { diff --git a/packages/cubejs-dremio-driver/package.json b/packages/cubejs-dremio-driver/package.json index 17d490a3bd5dc..52a0949d9276d 100644 --- a/packages/cubejs-dremio-driver/package.json +++ b/packages/cubejs-dremio-driver/package.json @@ -26,7 +26,6 @@ "@cubejs-backend/schema-compiler": "1.1.16", "@cubejs-backend/shared": "1.1.12", "axios": "^0.21.1", - "moment-timezone": "^0.5.31", "sqlstring": "^2.3.1" }, "devDependencies": { diff --git a/packages/cubejs-dremio-driver/test/DremioQuery.test.ts b/packages/cubejs-dremio-driver/test/DremioQuery.test.ts index 94dec6a15aeb3..618ab45415d29 100644 --- a/packages/cubejs-dremio-driver/test/DremioQuery.test.ts +++ b/packages/cubejs-dremio-driver/test/DremioQuery.test.ts @@ -63,7 +63,7 @@ cube(\`sales\`, { const queryAndParams = query.buildSqlAndParams(); expect(queryAndParams[0]).toContain( - 'DATE_TRUNC(\'day\', CONVERT_TIMEZONE(\'-08:00\', "sales".sales_datetime))' + 'DATE_TRUNC(\'day\', CONVERT_TIMEZONE(\'America/Los_Angeles\', "sales".sales_datetime))' ); })); diff --git a/packages/cubejs-druid-driver/package.json b/packages/cubejs-druid-driver/package.json index 3a9616e0297c1..7a1ce405b8a18 100644 --- a/packages/cubejs-druid-driver/package.json +++ b/packages/cubejs-druid-driver/package.json @@ -31,8 +31,7 @@ "@cubejs-backend/base-driver": "1.1.16", "@cubejs-backend/schema-compiler": "1.1.16", "@cubejs-backend/shared": "1.1.12", - "axios": "^0.21.1", - "moment-timezone": "^0.5.31" + "axios": "^0.21.1" }, "devDependencies": { "@cubejs-backend/linter": "^1.0.0", diff --git a/packages/cubejs-druid-driver/src/DruidQuery.ts b/packages/cubejs-druid-driver/src/DruidQuery.ts index c22abc450101b..e5d5235fba2d9 100644 --- a/packages/cubejs-druid-driver/src/DruidQuery.ts +++ b/packages/cubejs-druid-driver/src/DruidQuery.ts @@ -1,4 +1,3 @@ -import moment from 'moment-timezone'; import { BaseFilter, BaseQuery } from '@cubejs-backend/schema-compiler'; const GRANULARITY_TO_INTERVAL: Record string> = { diff --git a/packages/cubejs-query-orchestrator/package.json b/packages/cubejs-query-orchestrator/package.json index 754dbe5d20f3b..1a11a78d9b2dd 100644 --- a/packages/cubejs-query-orchestrator/package.json +++ b/packages/cubejs-query-orchestrator/package.json @@ -36,7 +36,6 @@ "es5-ext": "0.10.53", "generic-pool": "^3.7.1", "lru-cache": "^6.0.0", - "moment-timezone": "^0.5.33", "ramda": "^0.27.2" }, "devDependencies": { diff --git a/packages/cubejs-schema-compiler/package.json b/packages/cubejs-schema-compiler/package.json index 82856429dbb84..7d842e44f5f70 100644 --- a/packages/cubejs-schema-compiler/package.json +++ b/packages/cubejs-schema-compiler/package.json @@ -50,7 +50,7 @@ "joi": "^17.8.3", "js-yaml": "^4.1.0", "lru-cache": "^5.1.1", - "moment-timezone": "^0.5.33", + "moment-timezone": "^0.5.46", "node-dijkstra": "^2.5.0", "ramda": "^0.27.2", "syntax-error": "^1.3.0", diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 002586ce92307..f7154b203e0e6 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -2818,10 +2818,6 @@ export class BaseQuery { return this.join && this.join.multiplicationFactor[cubeName]; } - inIntegrationTimeZone(date) { - return moment.tz(date, this.timezone); - } - inDbTimeZone(date) { return inDbTimeZone(this.timezone, this.timestampFormat(), date); } diff --git a/packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts b/packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts index 535baf1eead09..afb51ee45fbc8 100644 --- a/packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts @@ -47,7 +47,7 @@ export class CubeStoreQuery extends BaseQuery { } public timestampFormat() { - return moment.HTML5_FMT.DATETIME_LOCAL_MS; + return 'YYYY-MM-DDTHH:mm:ss.SSS'; } public dateTimeCast(value) { diff --git a/packages/cubejs-schema-compiler/src/adapter/MysqlQuery.ts b/packages/cubejs-schema-compiler/src/adapter/MysqlQuery.ts index a6a304b7e9f01..3bb01cd09d5b8 100644 --- a/packages/cubejs-schema-compiler/src/adapter/MysqlQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/MysqlQuery.ts @@ -1,7 +1,5 @@ import moment from 'moment-timezone'; - -import { parseSqlInterval } from '@cubejs-backend/shared'; - +import { getEnv, parseSqlInterval } from '@cubejs-backend/shared'; import { BaseQuery } from './BaseQuery'; import { BaseFilter } from './BaseFilter'; import { UserError } from '../compiler/UserError'; @@ -26,39 +24,50 @@ class MysqlFilter extends BaseFilter { } export class MysqlQuery extends BaseQuery { + private readonly useNamedTimezones: boolean; + + public constructor(compilers: any, options: any) { + super(compilers, options); + + this.useNamedTimezones = getEnv('mysqlUseNamedTimezones', { dataSource: this.dataSource }); + } + public newFilter(filter) { return new MysqlFilter(this, filter); } - public castToString(sql) { + public castToString(sql: string) { return `CAST(${sql} as CHAR)`; } - public convertTz(field) { + public convertTz(field: string) { + if (this.useNamedTimezones) { + return `CONVERT_TZ(${field}, @@session.time_zone, '${this.timezone}')`; + } return `CONVERT_TZ(${field}, @@session.time_zone, '${moment().tz(this.timezone).format('Z')}')`; } - public timeStampCast(value) { + public timeStampCast(value: string) { return `TIMESTAMP(convert_tz(${value}, '+00:00', @@session.time_zone))`; } public timestampFormat() { - return moment.HTML5_FMT.DATETIME_LOCAL_MS; + return 'YYYY-MM-DDTHH:mm:ss.SSS'; } - public dateTimeCast(value) { + public dateTimeCast(value: string) { return `TIMESTAMP(${value})`; } - public subtractInterval(date, interval) { + public subtractInterval(date: string, interval: string) { return `DATE_SUB(${date}, INTERVAL ${this.formatInterval(interval)})`; } - public addInterval(date, interval) { + public addInterval(date: string, interval: string) { return `DATE_ADD(${date}, INTERVAL ${this.formatInterval(interval)})`; } - public timeGroupedColumn(granularity, dimension) { + public timeGroupedColumn(granularity: string, dimension) { return `CAST(${GRANULARITY_TO_INTERVAL[granularity](dimension)} AS DATETIME)`; } diff --git a/packages/cubejs-vertica-driver/package.json b/packages/cubejs-vertica-driver/package.json index a855dee258f6c..36e42fee556a6 100644 --- a/packages/cubejs-vertica-driver/package.json +++ b/packages/cubejs-vertica-driver/package.json @@ -22,7 +22,6 @@ "@cubejs-backend/base-driver": "1.1.16", "@cubejs-backend/query-orchestrator": "1.1.16", "@cubejs-backend/schema-compiler": "1.1.16", - "moment-timezone": "^0.5.45", "vertica-nodejs": "^1.0.3" }, "license": "Apache-2.0", diff --git a/packages/cubejs-vertica-driver/src/VerticaQuery.js b/packages/cubejs-vertica-driver/src/VerticaQuery.js index 8fd60bc4581f8..512178587cdb3 100644 --- a/packages/cubejs-vertica-driver/src/VerticaQuery.js +++ b/packages/cubejs-vertica-driver/src/VerticaQuery.js @@ -1,4 +1,3 @@ -const moment = require('moment-timezone'); const { BaseFilter, BaseQuery } = require('@cubejs-backend/schema-compiler'); const GRANULARITY_TO_INTERVAL = { @@ -38,7 +37,7 @@ class VerticaQuery extends BaseQuery { } timestampFormat() { - return moment.HTML5_FMT.DATETIME_LOCAL_MS; + return 'YYYY-MM-DDTHH:mm:ss.SSS'; } dateTimeCast(value) { diff --git a/yarn.lock b/yarn.lock index bc41c494cdb70..97afdc40554b4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -21779,16 +21779,16 @@ moment-range@*, moment-range@^4.0.1: dependencies: es6-symbol "^3.1.0" -moment-timezone@^0.5.15, moment-timezone@^0.5.27, moment-timezone@^0.5.31, moment-timezone@^0.5.33: +moment-timezone@^0.5.15, moment-timezone@^0.5.33: version "0.5.45" resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.45.tgz#cb685acd56bac10e69d93c536366eb65aa6bcf5c" integrity sha512-HIWmqA86KcmCAhnMAN0wuDOARV/525R2+lOLotuGFzn4HO+FH+/645z2wx0Dt3iDv6/p61SIvKnDstISainhLQ== dependencies: moment "^2.29.4" -moment-timezone@^0.5.45: +moment-timezone@^0.5.46: version "0.5.46" - resolved "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.46.tgz#a21aa6392b3c6b3ed916cd5e95858a28d893704a" + resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.46.tgz#a21aa6392b3c6b3ed916cd5e95858a28d893704a" integrity sha512-ZXm9b36esbe7OmdABqIWJuBBiLLwAjrN7CE+7sYdCCx82Nabt1wHDj8TVseS59QIlfFPbOoiBPm6ca9BioG4hw== dependencies: moment "^2.29.4" From db3b0fd6147f2b0889432eaf8b3f6d45a50461ec Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 24 Jan 2025 17:27:24 +0200 Subject: [PATCH 4/5] feat(snowflake-driver): Add ignore case statements flag (#9131) * Fixing case statements in snowflake driver * Fixes case sensitivity for snowflake to be default and env driven Adds env var for the snowflake driver to enable or disable the case sensitivity and if not set will default to case insensitive * Updating snowflake driver to respect case by default with added override. * fix types for identIgnoreCase in snowflake driver * fix --------- Co-authored-by: Micheal Taylor --- .../configuration/data-sources/snowflake.mdx | 6 ++++ packages/cubejs-backend-shared/src/env.ts | 33 +++++++++++++++++++ .../src/SnowflakeDriver.ts | 8 +++++ 3 files changed, 47 insertions(+) diff --git a/docs/pages/product/configuration/data-sources/snowflake.mdx b/docs/pages/product/configuration/data-sources/snowflake.mdx index 32d457ee32068..d27de799c56dd 100644 --- a/docs/pages/product/configuration/data-sources/snowflake.mdx +++ b/docs/pages/product/configuration/data-sources/snowflake.mdx @@ -12,6 +12,12 @@ redirect_from: - [The region][snowflake-docs-regions] for the [Snowflake][snowflake] warehouse - The username/password for the [Snowflake][snowflake] account +## Snowflake quoted identifiers + +Due to an issue in snowflakes opinion about quoted identifers we set a session value to override +snowflake defaults for users that have set an account value for: QUOTED_IDENTIFIERS_IGNORE_CASE +you can learn more about this here: https://docs.snowflake.com/en/sql-reference/identifiers-syntax#double-quoted-identifiers + ## Setup diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index 36a4bf4e7c41b..4d4a2e97462a3 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -1529,6 +1529,39 @@ const variables: Record any> = { ] ), + /** + * Snowflake case sensitivity for identifiers (like database columns). + */ + snowflakeQuotedIdentIgnoreCase: ({ + dataSource + }: { + dataSource: string, + }) => { + const val = process.env[ + keyByDataSource( + 'CUBEJS_DB_SNOWFLAKE_QUOTED_IDENTIFIERS_IGNORE_CASE', + dataSource, + ) + ]; + if (val) { + if (val.toLocaleLowerCase() === 'true') { + return true; + } else if (val.toLowerCase() === 'false') { + return false; + } else { + throw new TypeError( + `The ${ + keyByDataSource( + 'CUBEJS_DB_SNOWFLAKE_QUOTED_IDENTIFIERS_IGNORE_CASE', + dataSource, + ) + } must be either 'true' or 'false'.` + ); + } + } else { + return false; + } + }, /** **************************************************************** * Presto Driver * ***************************************************************** */ diff --git a/packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts b/packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts index ccf19191fa980..99a682ea31388 100644 --- a/packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts +++ b/packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts @@ -173,6 +173,7 @@ interface SnowflakeDriverOptions { resultPrefetch?: number, exportBucket?: SnowflakeDriverExportBucket, executionTimeout?: number, + identIgnoreCase?: boolean, application: string, readOnly?: boolean, @@ -213,6 +214,7 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface { 'CUBEJS_DB_SNOWFLAKE_PRIVATE_KEY_PATH', 'CUBEJS_DB_SNOWFLAKE_PRIVATE_KEY_PASS', 'CUBEJS_DB_SNOWFLAKE_OAUTH_TOKEN_PATH', + 'CUBEJS_DB_SNOWFLAKE_QUOTED_IDENTIFIERS_IGNORE_CASE', ]; } @@ -279,6 +281,7 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface { exportBucket: this.getExportBucket(dataSource), resultPrefetch: 1, executionTimeout: getEnv('dbQueryTimeout', { dataSource }), + identIgnoreCase: getEnv('snowflakeQuotedIdentIgnoreCase', { dataSource }), exportBucketCsvEscapeSymbol: getEnv('dbExportBucketCsvEscapeSymbol', { dataSource }), application: 'CubeDev_Cube', ...config @@ -451,6 +454,11 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface { await this.execute(connection, 'ALTER SESSION SET TIMEZONE = \'UTC\'', [], false); await this.execute(connection, `ALTER SESSION SET STATEMENT_TIMEOUT_IN_SECONDS = ${this.config.executionTimeout}`, [], false); + // We only want to ignore the case if someone sets the value to true explicitly + // since the default assumption is that casing matters + if (this.config.identIgnoreCase) { + await this.execute(connection, 'ALTER SESSION SET QUOTED_IDENTIFIERS_IGNORE_CASE = TRUE', [], false); + } return connection; } catch (e) { this.connection = null; From 382210716fc3c9ed459c5b45a8a52e766ff7d7cf Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 24 Jan 2025 19:29:17 +0200 Subject: [PATCH 5/5] feat(api-gateway): Async native query results transformations (#8961) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * create a basic cubeorchestrator project structure * wip * move flatbuffer schema/code to separate crate * implement parse_cubestore_ws_result * add cubeorchestrator/parse_cubestore_ws_result_message export * use native parseCubestoreResultMessage * init hashmap with capacity * cargo fmt * some optimizations and improvements * a bit optimized version * use cx.execute_scoped for optimization * a bit more rust idiomatic code * put native parseCubestoreResultMessage behind the flag * tiny improvement * cargo fmt * cargo fmt * cargo clippy fix * update cubestore Dockerfile * cargo fmt * update cubestore Docker builds * introduce CubeStoreResult struct * create CubeStoreResultWrapper class and switch to lazy evaluation of results set * add resToRawResultFn in API GW * cargo fmt * update resToResultFn * call prepareAnnotation later * remove bytes * add cached flag to CubeStoreResultWrapper * convert core data types from api gateway * implement transformData and related helpers * cargo fmt * down chrono to same version as in cubesql (0.4.31) * fail fast in api gw load() * update cargo.lock (syn crate) * linter fix * implement get_query_granularities & get_pivot_query * prepare transformQueryData native wrapper * small optimization: do not use native parsing for short messages * refactor transformValue and related * types restructure * debug and fix native transform_data() * lazy transformData evaluation * omplement get_final_cubestore_result & get_final_cubestore_result_multi in native * cargo fmt * cargo clippy fix * refactor getVanillaRow * implement get_final_cubestore_result_array() native * fix native response flow for sqlApiLoad * add postgres with native cubestore results driver tests * Build native (without Python) in drivers tests * workaround for native build in testings-drivers * small improvements in CubeStoreResultWrapper * make parse_cubestore_ws_result_message async * make all native cubestore_result_transform functions async * cargo fmt * refactor results transformations * yarn sync * implement custom deserializer from JS Value to rust and updated the cubestore result transformations with it * refactor json_to_array_buffer() * a bit of refactoring * code rearrangement * switch to use DBResponsePrimitive instead of just Strings * refactoring * always use transform data native for all results * fix dtos for native query processing (thnx unit tests) * cache loadNative * remove not needed anymore native.transformQueryData * add Build native for unit tests in CI * refactor serde annotations * remove unused * annotate native query results processing functions * add Build native for unit tests in CI for Debian without pushing * add few unit tests * fix some tests * attempt to fix native build/test in docker-dev CI * fix empty result set issue * another fix in datetime parsing * another fix in deserialization * another fix in datetime parsing * update postgres-native-cubestore-response-full.test.ts.snap * another fix in deserialization * cargo fmt * another fix in datetime parsing * update postgres-native-cubestore-response-full.test.ts.snap * attempt to fix native build/test in docker-dev CI * add some comments * edits in result processing when streaming * fix native response processing via websocket * fix yarn lock * attempt to fix native build/test in cloud integration tests in CI * recreated unit tests for transform data in native * run rust unit tests on push * commented out cargo fmt/build/test for cubesql, cubesqlplanner * remove transformdata JS implementation * refactor push CI jobs * rename CubestoreResultWrapper → ResultWrapper * add isNative flag to ResultWrapper * rename getResultInternal → prepareResultTransformData * use ResultWrapper for all results * fix getArray in ResultWrapper * lint fix * encapsulate rootResultObject into ResultWrapper * add DataResult interface + implement it in result wrappers * add isWrapper property to wrappers * transform wrapped result later before returning the response * fix lint warn * fix async request porcessing * some node version fix * weird yarn.lock change. Wtf? * full transition of result native-js-native * fix @cubejs-backend/native version ref * refactor ResultWrapper constructor * fix string deserialize in transport * extend DataResult interface * support iterative access in ResultWrapper * refactor ResultWrapper classes * pass ResultWrapper to native and offload transformData from the eventloop * remove obsolete getFinalQueryResultArray * fix ws api subscriptionServer results processing * pass SchemaRef to load methods. * trying to fix flaky ws failing tests * remove cubeorchestrator dependency from cubesql * rewrite all load api calls to return RecordBatch * fix @cubejs-backend/native version ref after rebase * set prototype of ResultWrapper proxy to class prototype * fix linter warnings * fix yarn.lock * linter fix * Set CUBEJS_TESSERACT_ORCHESTRATOR: true for all tests workflows * fix old refs after rebase * update integration-cubestore job with the build of backend-native * remove postgres-native-cubestore-response from testing drivers as we turnon new orchestrator for all tests --- .github/workflows/cloud.yml | 21 + .github/workflows/drivers-tests.yml | 14 +- .github/workflows/master.yml | 2 + .github/workflows/publish.yml | 2 +- .github/workflows/push.yml | 123 +- .github/workflows/rust-cubestore-master.yml | 2 +- .github/workflows/rust-cubestore.yml | 2 +- .../src/SubscriptionServer.ts | 4 +- packages/cubejs-api-gateway/src/gateway.ts | 149 +- .../src/helpers/prepareAnnotation.ts | 1 + .../src/helpers/toConfigMap.ts | 2 +- .../src/helpers/transformData.ts | 373 --- .../src/helpers/transformValue.ts | 48 - packages/cubejs-api-gateway/src/index.ts | 1 + packages/cubejs-api-gateway/src/interfaces.ts | 16 + .../cubejs-api-gateway/src/types/request.ts | 3 +- .../cubejs-api-gateway/src/types/responses.ts | 35 + .../test/helpers/transformData.test.ts | 1339 --------- .../test/helpers/transformValue.test.ts | 33 - packages/cubejs-backend-native/Cargo.lock | 119 +- packages/cubejs-backend-native/Cargo.toml | 6 +- .../cubejs-backend-native/js/ResultWrapper.ts | 237 ++ packages/cubejs-backend-native/js/index.ts | 71 +- packages/cubejs-backend-native/src/channel.rs | 10 +- packages/cubejs-backend-native/src/lib.rs | 2 + .../cubejs-backend-native/src/node_export.rs | 13 +- .../src/node_obj_deserializer.rs | 436 +++ .../cubejs-backend-native/src/orchestrator.rs | 346 +++ .../cubejs-backend-native/src/transport.rs | 143 +- .../cubejs-backend-native/test/sql.test.ts | 2 +- packages/cubejs-backend-shared/package.json | 2 +- packages/cubejs-backend-shared/src/env.ts | 3 + packages/cubejs-cubestore-driver/package.json | 1 + .../src/WebSocketConnection.ts | 84 +- packages/cubejs-mssql-driver/package.json | 2 +- .../cubejs-server/src/websocket-server.ts | 18 +- rust/cubenativeutils/Cargo.lock | 55 +- rust/cubenativeutils/rustfmt.toml | 1 + rust/cubenativeutils/src/wrappers/context.rs | 3 +- .../src/wrappers/inner_types.rs | 10 +- .../src/wrappers/neon/context.rs | 27 +- .../src/wrappers/neon/inner_types.rs | 13 +- .../src/wrappers/neon/object/mod.rs | 13 +- .../src/wrappers/neon/object/neon_array.rs | 8 +- .../src/wrappers/neon/object/neon_function.rs | 8 +- .../src/wrappers/neon/object/neon_struct.rs | 8 +- rust/cubenativeutils/src/wrappers/object.rs | 3 +- .../src/wrappers/object_handle.rs | 3 +- .../src/wrappers/serializer/deserialize.rs | 7 +- .../src/wrappers/serializer/deserializer.rs | 17 +- .../src/wrappers/serializer/error.rs | 3 +- .../src/wrappers/serializer/serialize.rs | 7 +- .../src/wrappers/serializer/serializer.rs | 7 +- rust/cubeorchestrator/.gitignore | 3 + rust/cubeorchestrator/CHANGELOG.md | 2 + rust/cubeorchestrator/Cargo.lock | 537 ++++ rust/cubeorchestrator/Cargo.toml | 17 + rust/cubeorchestrator/rust-toolchain.toml | 5 + rust/cubeorchestrator/rustfmt.toml | 1 + rust/cubeorchestrator/src/lib.rs | 3 + .../src/query_message_parser.rs | 146 + .../src/query_result_transform.rs | 2500 +++++++++++++++++ rust/cubeorchestrator/src/transport.rs | 290 ++ rust/cubeshared/.gitignore | 3 + rust/cubeshared/CHANGELOG.md | 2 + rust/cubeshared/Cargo.lock | 41 + rust/cubeshared/Cargo.toml | 7 + rust/cubeshared/flatbuffers-codegen.sh | 4 + rust/cubeshared/rust-toolchain.toml | 5 + rust/cubeshared/rustfmt.toml | 1 + .../src/codegen/http_message.fbs | 0 .../src/codegen/http_message_generated.rs | 0 .../src/codegen/mod.rs | 1 + rust/cubeshared/src/lib.rs | 1 + rust/cubesql/Cargo.lock | 38 +- .../cubeclient/src/models/v1_load_result.rs | 1 - .../cubesql/src/compile/engine/df/scan.rs | 141 +- .../cubesql/src/compile/engine/udf/common.rs | 7 +- .../src/compile/rewrite/rules/filters.rs | 4 +- rust/cubesql/cubesql/src/compile/test/mod.rs | 18 +- rust/cubesql/cubesql/src/transport/service.rs | 16 +- rust/cubesqlplanner/Cargo.lock | 55 +- rust/cubestore/Cargo.lock | 8 + rust/cubestore/Dockerfile | 25 +- rust/cubestore/cubestore/Cargo.toml | 1 + rust/cubestore/cubestore/src/http/mod.rs | 30 +- rust/cubestore/cubestore/src/lib.rs | 1 - 87 files changed, 5531 insertions(+), 2241 deletions(-) delete mode 100644 packages/cubejs-api-gateway/src/helpers/transformData.ts delete mode 100644 packages/cubejs-api-gateway/src/helpers/transformValue.ts create mode 100644 packages/cubejs-api-gateway/src/types/responses.ts delete mode 100644 packages/cubejs-api-gateway/test/helpers/transformData.test.ts delete mode 100644 packages/cubejs-api-gateway/test/helpers/transformValue.test.ts create mode 100644 packages/cubejs-backend-native/js/ResultWrapper.ts create mode 100644 packages/cubejs-backend-native/src/node_obj_deserializer.rs create mode 100644 packages/cubejs-backend-native/src/orchestrator.rs create mode 100644 rust/cubenativeutils/rustfmt.toml create mode 100644 rust/cubeorchestrator/.gitignore create mode 100644 rust/cubeorchestrator/CHANGELOG.md create mode 100644 rust/cubeorchestrator/Cargo.lock create mode 100644 rust/cubeorchestrator/Cargo.toml create mode 100644 rust/cubeorchestrator/rust-toolchain.toml create mode 100644 rust/cubeorchestrator/rustfmt.toml create mode 100644 rust/cubeorchestrator/src/lib.rs create mode 100644 rust/cubeorchestrator/src/query_message_parser.rs create mode 100644 rust/cubeorchestrator/src/query_result_transform.rs create mode 100644 rust/cubeorchestrator/src/transport.rs create mode 100644 rust/cubeshared/.gitignore create mode 100644 rust/cubeshared/CHANGELOG.md create mode 100644 rust/cubeshared/Cargo.lock create mode 100644 rust/cubeshared/Cargo.toml create mode 100755 rust/cubeshared/flatbuffers-codegen.sh create mode 100644 rust/cubeshared/rust-toolchain.toml create mode 100644 rust/cubeshared/rustfmt.toml rename rust/{cubestore/cubestore => cubeshared}/src/codegen/http_message.fbs (100%) rename rust/{cubestore/cubestore => cubeshared}/src/codegen/http_message_generated.rs (100%) rename rust/{cubestore/cubestore => cubeshared}/src/codegen/mod.rs (71%) create mode 100644 rust/cubeshared/src/lib.rs diff --git a/.github/workflows/cloud.yml b/.github/workflows/cloud.yml index 81ea469639f83..04e2ff0087425 100644 --- a/.github/workflows/cloud.yml +++ b/.github/workflows/cloud.yml @@ -25,6 +25,9 @@ on: - 'package.json' - 'yarn.lock' +env: + CUBEJS_TESSERACT_ORCHESTRATOR: true + jobs: latest-tag-sha: runs-on: ubuntu-20.04 @@ -59,15 +62,31 @@ jobs: matrix: node-version: [ 20.x ] db: [ 'athena', 'bigquery', 'snowflake' ] + target: [ "x86_64-unknown-linux-gnu" ] fail-fast: false steps: - name: Checkout uses: actions/checkout@v4 + - name: Install Rust + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: nightly-2024-07-15 + # override: true # this is by default on + rustflags: "" + components: rustfmt + target: ${{ matrix.target }} - name: Install Node.js ${{ matrix.node-version }} uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} + - name: Install cargo-cp-artifact + run: npm install -g cargo-cp-artifact@0.1 + - uses: Swatinem/rust-cache@v2 + with: + workspaces: ./packages/cubejs-backend-native + key: native-${{ runner.OS }}-${{ matrix.target }} + shared-key: native-${{ runner.OS }}-${{ matrix.target }} - name: Get yarn cache directory path id: yarn-cache-dir-path run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT" @@ -87,6 +106,8 @@ jobs: run: yarn build - name: Lerna tsc run: yarn tsc + - name: Build native (no python) + run: cd packages/cubejs-backend-native && npm run native:build-release - name: Run Integration tests for ${{ matrix.db }} matrix timeout-minutes: 30 env: diff --git a/.github/workflows/drivers-tests.yml b/.github/workflows/drivers-tests.yml index 1ab0154df375b..3cec502954c68 100644 --- a/.github/workflows/drivers-tests.yml +++ b/.github/workflows/drivers-tests.yml @@ -56,6 +56,9 @@ on: - 'packages/cubejs-backend-native/**' - 'rust/cubesql/**' +env: + CUBEJS_TESSERACT_ORCHESTRATOR: true + jobs: latest-tag-sha: runs-on: ubuntu-20.04 @@ -117,8 +120,8 @@ jobs: - uses: Swatinem/rust-cache@v2 with: workspaces: ./packages/cubejs-backend-native - key: native-${{ runner.OS }}-x86_64-unknown-linux-gnu - shared-key: native-${{ runner.OS }}-x86_64-unknown-linux-gnu + key: native-${{ runner.OS }}-${{ matrix.target }} + shared-key: native-${{ runner.OS }}-${{ matrix.target }} - name: Build native (fallback) if: (matrix.python-version == 'fallback') env: @@ -165,7 +168,12 @@ jobs: uses: actions/download-artifact@v4 with: name: backend-native - path: packages/cubejs-backend-native/ + path: packages/cubejs-backend-native + # current .dockerignore prevents use of native build + - name: Unignore native from .dockerignore + run: | + grep -v -E "packages/cubejs-backend-native/((native)|(index.node))" .dockerignore > .dockerignore.tmp + mv .dockerignore.tmp .dockerignore - name: Build and push uses: docker/build-push-action@v6 with: diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index d1fbc157e5b47..9b0c92d7269d1 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -16,6 +16,8 @@ on: - 'rust/cubesql/**' branches: - master +env: + CUBEJS_TESSERACT_ORCHESTRATOR: true jobs: latest-tag-sha: runs-on: ubuntu-20.04 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 92d896a6421bd..3c3d0e70a329f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -552,7 +552,7 @@ jobs: - name: Push to Docker Hub uses: docker/build-push-action@v6 with: - context: ./rust/cubestore/ + context: ./rust/ file: ./rust/cubestore/Dockerfile platforms: ${{ matrix.platforms }} build-args: ${{ matrix.build-args }} diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 057532433495d..25d30e269e617 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -9,6 +9,10 @@ on: - 'packages/**' - 'rust/cubestore/**' - 'rust/cubesql/**' + - 'rust/cubenativeutils/**' + - 'rust/cubeorchestrator/**' + - 'rust/cubeshared/**' + - 'rust/cubesqlplanner/**' - '.eslintrc.js' - '.prettierrc' - 'package.json' @@ -24,6 +28,10 @@ on: - 'packages/**' - 'rust/cubestore/**' - 'rust/cubesql/**' + - 'rust/cubenativeutils/**' + - 'rust/cubeorchestrator/**' + - 'rust/cubeshared/**' + - 'rust/cubesqlplanner/**' - '.eslintrc.js' - '.prettierrc' - 'package.json' @@ -31,6 +39,9 @@ on: - 'rollup.config.js' - 'yarn.lock' +env: + CUBEJS_TESSERACT_ORCHESTRATOR: true + jobs: unit: runs-on: ubuntu-20.04 @@ -95,6 +106,8 @@ jobs: command: yarn install --frozen-lockfile - name: Lerna tsc run: yarn tsc + - name: Build native (no python) + run: cd packages/cubejs-backend-native && npm run native:build-release - name: Build client run: yarn build - name: Build cubejs-backend-native (with Python) @@ -102,7 +115,6 @@ jobs: working-directory: ./packages/cubejs-backend-native env: PYO3_PYTHON: python${{ matrix.python-version }} - - name: Lerna test run: yarn lerna run --concurrency 1 --stream --no-prefix unit # - uses: codecov/codecov-action@v1 @@ -111,6 +123,21 @@ jobs: # files: ./packages/*/coverage/clover.xml # flags: cube-backend # verbose: true # optional (default = false) + - name: Cargo test cubeorchestrator + run: | + cargo test --manifest-path rust/cubeorchestrator/Cargo.toml -j 1 + - name: Cargo test cubenativeutils + run: | + cargo test --manifest-path rust/cubenativeutils/Cargo.toml -j 1 + - name: Cargo test cubeshared + run: | + cargo test --manifest-path rust/cubeshared/Cargo.toml -j 1 +# - name: Cargo test cubesql +# run: | +# cargo test --manifest-path rust/cubesql/Cargo.toml -j 1 +# - name: Cargo test cubesqlplanner +# run: | +# cargo test --manifest-path rust/cubesqlplanner/cubesqlplanner/Cargo.toml -j 1 lint: runs-on: ubuntu-20.04 @@ -159,6 +186,21 @@ jobs: run: yarn lint:npm - name: Lerna lint run: yarn lerna run --concurrency 1 lint + - name: Cargo fmt cubeorchestrator + run: | + cargo fmt --manifest-path rust/cubeorchestrator/Cargo.toml -- --check + - name: Cargo fmt cubenativeutils + run: | + cargo fmt --manifest-path rust/cubenativeutils/Cargo.toml -- --check + - name: Cargo fmt cubeshared + run: | + cargo fmt --manifest-path rust/cubeshared/Cargo.toml -- --check +# - name: Cargo fmt cubesql +# run: | +# cargo fmt --manifest-path rust/cubesql/Cargo.toml -- --check +# - name: Cargo fmt cubesqlplanner +# run: | +# cargo fmt --manifest-path rust/cubesqlplanner/cubesqlplanner/Cargo.toml -- --check build: runs-on: ubuntu-20.04 @@ -211,6 +253,21 @@ jobs: run: yarn lerna run --concurrency 1 build env: NODE_OPTIONS: --max_old_space_size=4096 + - name: Cargo build cubeorchestrator + run: | + cargo build --manifest-path rust/cubeorchestrator/Cargo.toml -j 4 + - name: Cargo build cubenativeutils + run: | + cargo build --manifest-path rust/cubenativeutils/Cargo.toml -j 4 + - name: Cargo build cubeshared + run: | + cargo build --manifest-path rust/cubeshared/Cargo.toml -j 4 +# - name: Cargo build cubesql +# run: | +# cargo build --manifest-path rust/cubesql/Cargo.toml -j 4 +# - name: Cargo build cubesqlplanner +# run: | +# cargo build --manifest-path rust/cubesqlplanner/cubesqlplanner/Cargo.toml -j 4 build-cubestore: needs: [latest-tag-sha] @@ -276,6 +333,13 @@ jobs: df -h - name: Checkout uses: actions/checkout@v4 + - name: Install Rust + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: nightly-2024-07-15 + # override: true # this is by default on + rustflags: "" + components: rustfmt - name: Install Node.js ${{ matrix.node-version }} uses: actions/setup-node@v4 with: @@ -305,6 +369,9 @@ jobs: command: yarn install --frozen-lockfile - name: Lerna tsc run: yarn tsc + - name: Build cubejs-backend-native (without Python) + run: yarn run native:build-release + working-directory: ./packages/cubejs-backend-native - name: Download cubestored-x86_64-unknown-linux-gnu-release artifact uses: actions/download-artifact@v4 with: @@ -539,6 +606,8 @@ jobs: - 5000:5000 strategy: matrix: + node-version: [ 20 ] + target: [ "x86_64-unknown-linux-gnu" ] dockerfile: - dev.Dockerfile include: @@ -565,21 +634,29 @@ jobs: df -h - name: Checkout uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Build image - uses: docker/build-push-action@v6 - timeout-minutes: 30 + - name: Install Rust + uses: actions-rust-lang/setup-rust-toolchain@v1 with: - context: . - file: ./packages/cubejs-docker/${{ matrix.dockerfile }} - platforms: linux/amd64 - push: true - tags: localhost:5000/cubejs/cube:${{ matrix.tag }} - - name: Use Node.js 20.x + toolchain: nightly-2024-07-15 + # override: true # this is by default on + rustflags: "" + components: rustfmt + target: ${{ matrix.target }} + - name: Install Node.js ${{ matrix.node-version }} uses: actions/setup-node@v4 with: - node-version: 20.x + node-version: ${{ matrix.node-version }} + - name: Install Yarn + run: npm install -g yarn + - name: Set Yarn version + run: yarn policies set-version v1.22.22 + - name: Install cargo-cp-artifact + run: npm install -g cargo-cp-artifact@0.1 + - uses: Swatinem/rust-cache@v2 + with: + workspaces: ./packages/cubejs-backend-native + key: native-${{ runner.OS }}-${{ matrix.target }} + shared-key: native-${{ runner.OS }}-${{ matrix.target }} - name: Get yarn cache directory path id: yarn-cache-dir-path run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT" @@ -591,8 +668,6 @@ jobs: key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} restore-keys: | ${{ runner.os }}-yarn- - - name: Set Yarn version - run: yarn policies set-version v1.22.22 - name: Yarn install uses: nick-fields/retry@v3 env: @@ -607,6 +682,24 @@ jobs: run: yarn build - name: Lerna tsc run: yarn tsc + - name: Build native (no python) + run: cd packages/cubejs-backend-native && npm run native:build-release + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + # current .dockerignore prevents use of native build + - name: Unignore native from .dockerignore + run: | + grep -v -E "packages/cubejs-backend-native/((native)|(index.node))" .dockerignore > .dockerignore.tmp + mv .dockerignore.tmp .dockerignore + - name: Build image + uses: docker/build-push-action@v6 + timeout-minutes: 30 + with: + context: . + file: ./packages/cubejs-docker/${{ matrix.dockerfile }} + platforms: linux/amd64 + push: true + tags: localhost:5000/cubejs/cube:${{ matrix.tag }} - name: Testing CubeJS (container mode) via BirdBox run: | cd packages/cubejs-testing/ diff --git a/.github/workflows/rust-cubestore-master.yml b/.github/workflows/rust-cubestore-master.yml index 29211f7c24a83..eb72e456f30ea 100644 --- a/.github/workflows/rust-cubestore-master.yml +++ b/.github/workflows/rust-cubestore-master.yml @@ -151,7 +151,7 @@ jobs: - name: Push to Docker Hub uses: docker/build-push-action@v6 with: - context: ./rust/cubestore + context: ./rust file: ./rust/cubestore/Dockerfile platforms: ${{ matrix.platforms }} build-args: ${{ matrix.build-args }} diff --git a/.github/workflows/rust-cubestore.yml b/.github/workflows/rust-cubestore.yml index 3b4a2eb6182c3..05f443a705cc5 100644 --- a/.github/workflows/rust-cubestore.yml +++ b/.github/workflows/rust-cubestore.yml @@ -99,7 +99,7 @@ jobs: - name: Build only uses: docker/build-push-action@v6 with: - context: ./rust/cubestore/ + context: ./rust/ file: ./rust/cubestore/Dockerfile platforms: ${{ matrix.platforms }} build-args: ${{ matrix.build-args }} diff --git a/packages/cubejs-api-gateway/src/SubscriptionServer.ts b/packages/cubejs-api-gateway/src/SubscriptionServer.ts index 9cf7a17b32886..c45c302915d96 100644 --- a/packages/cubejs-api-gateway/src/SubscriptionServer.ts +++ b/packages/cubejs-api-gateway/src/SubscriptionServer.ts @@ -19,7 +19,7 @@ const calcMessageLength = (message: unknown) => Buffer.byteLength( typeof message === 'string' ? message : JSON.stringify(message) ); -export type WebSocketSendMessageFn = (connectionId: string, message: any) => void; +export type WebSocketSendMessageFn = (connectionId: string, message: any) => Promise; export class SubscriptionServer { public constructor( @@ -31,7 +31,7 @@ export class SubscriptionServer { } public resultFn(connectionId: string, messageId: string, requestId: string | undefined) { - return (message, { status } = { status: 200 }) => { + return async (message, { status } = { status: 200 }) => { this.apiGateway.log({ type: 'Outgoing network usage', service: 'api-ws', diff --git a/packages/cubejs-api-gateway/src/gateway.ts b/packages/cubejs-api-gateway/src/gateway.ts index 006db77b91d3b..af90986b4117b 100644 --- a/packages/cubejs-api-gateway/src/gateway.ts +++ b/packages/cubejs-api-gateway/src/gateway.ts @@ -10,6 +10,11 @@ import { getRealType, QueryAlias, } from '@cubejs-backend/shared'; +import { + ResultArrayWrapper, + ResultMultiWrapper, + ResultWrapper, +} from '@cubejs-backend/native'; import type { Application as ExpressApplication, ErrorRequestHandler, @@ -82,7 +87,6 @@ import { createJWKsFetcher } from './jwk'; import { SQLServer, SQLServerConstructorOptions } from './sql-server'; import { getJsonQueryFromGraphQLQuery, makeSchema } from './graphql'; import { ConfigItem, prepareAnnotation } from './helpers/prepareAnnotation'; -import transformData from './helpers/transformData'; import { transformCube, transformMeasure, @@ -1541,7 +1545,7 @@ class ApiGateway { context: RequestContext, normalizedQuery: NormalizedQuery, sqlQuery: any, - ) { + ): Promise { const queries = [{ ...sqlQuery, query: sqlQuery.sql[0], @@ -1586,15 +1590,28 @@ class ApiGateway { response.total = normalizedQuery.total ? Number(total.data[0][QueryAlias.TOTAL_COUNT]) : undefined; - return response; + + return this.wrapAdapterQueryResultIfNeeded(response); + } + + /** + * Wraps the adapter's response in unified ResultWrapper if it comes from + * a common driver (not a Cubestore's one, cause Cubestore Driver internally creates ResultWrapper) + * @param res Adapter's response + * @private + */ + private wrapAdapterQueryResultIfNeeded(res: any): ResultWrapper { + res.data = new ResultWrapper(res.data); + + return res; } /** - * Convert adapter's result and other request paramters to a final + * Prepare adapter's result and other transform parameters for a final * result object. * @internal */ - private getResultInternal( + private prepareResultTransformData( context: RequestContext, queryType: QueryType, normalizedQuery: NormalizedQuery, @@ -1615,21 +1632,23 @@ class ApiGateway { }, response: any, responseType?: ResultType, - ) { - return { + ): ResultWrapper { + const resultWrapper = response.data; + + const transformDataParams = { + aliasToMemberNameMap: sqlQuery.aliasNameToMember, + annotation: { + ...annotation.measures, + ...annotation.dimensions, + ...annotation.timeDimensions + } as { [member: string]: ConfigItem }, + query: normalizedQuery, + queryType, + resType: responseType, + }; + + const resObj = { query: normalizedQuery, - data: transformData( - sqlQuery.aliasNameToMember, - { - ...annotation.measures, - ...annotation.dimensions, - ...annotation.timeDimensions - } as { [member: string]: ConfigItem }, - response.data, - normalizedQuery, - queryType, - responseType, - ), lastRefreshTime: response.lastRefreshTime?.toISOString(), ...( getEnv('devMode') || @@ -1650,6 +1669,11 @@ class ApiGateway { slowQuery: Boolean(response.slowQuery), total: normalizedQuery.total ? response.total : null, }; + + resultWrapper.setTransformData(transformDataParams); + resultWrapper.setRootResultObject(resObj); + + return resultWrapper; } /** @@ -1733,6 +1757,17 @@ class ApiGateway { const [queryType, normalizedQueries] = await this.getNormalizedQueries(query, context); + if ( + queryType !== QueryTypeEnum.REGULAR_QUERY && + props.queryType == null + ) { + throw new UserError( + `'${queryType + }' query type is not supported by the client.` + + 'Please update the client.' + ); + } + let metaConfigResult = await (await this .getCompilerApi(context)).metaConfig(request.context, { requestId: context.requestId @@ -1749,17 +1784,17 @@ class ApiGateway { slowQuery = slowQuery || Boolean(sqlQueries[index].slowQuery); - const annotation = prepareAnnotation( - metaConfigResult, normalizedQuery - ); - const response = await this.getSqlResponseInternal( context, normalizedQuery, sqlQueries[index], ); - return this.getResultInternal( + const annotation = prepareAnnotation( + metaConfigResult, normalizedQuery + ); + + return this.prepareResultTransformData( context, queryType, normalizedQuery, @@ -1783,37 +1818,24 @@ class ApiGateway { queries: results.length, queriesWithPreAggregations: results.filter( - (r: any) => Object.keys( - r.usedPreAggregations || {} - ).length + (r: any) => Object.keys(r.getRootResultObject()[0].usedPreAggregations || {}).length ).length, - queriesWithData: - results.filter((r: any) => r.data?.length).length, - dbType: results.map(r => r.dbType), + // Have to omit because data could be processed natively + // so it is not known at this point + // queriesWithData: + // results.filter((r: any) => r.data?.length).length, + dbType: results.map(r => r.getRootResultObject()[0].dbType), }, context, ); - if ( - queryType !== QueryTypeEnum.REGULAR_QUERY && - props.queryType == null - ) { - throw new UserError( - `'${queryType - }' query type is not supported by the client.` + - 'Please update the client.' - ); - } - if (props.queryType === 'multi') { - res({ - queryType, - results, - pivotQuery: getPivotQuery(queryType, normalizedQueries), - slowQuery - }); + // We prepare the final json result on native side + const resultMulti = new ResultMultiWrapper(results, { queryType, slowQuery }); + await res(resultMulti); } else { - res(results[0]); + // We prepare the full final json result on native side + await res(results[0]); } } catch (e: any) { this.handleError({ @@ -1909,6 +1931,8 @@ class ApiGateway { annotation }]; } + + await res(request.streaming ? results[0] : { results }); } else { results = await Promise.all( normalizedQueries.map(async (normalizedQuery, index) => { @@ -1929,7 +1953,7 @@ class ApiGateway { sqlQueries[index], ); - return this.getResultInternal( + return this.prepareResultTransformData( context, queryType, normalizedQuery, @@ -1940,11 +1964,15 @@ class ApiGateway { ); }) ); - } - res(request.streaming ? results[0] : { - results, - }); + if (request.streaming) { + await res(results[0]); + } else { + // We prepare the final json result on native side + const resultArray = new ResultArrayWrapper(results); + await res(resultArray); + } + } } catch (e: any) { this.handleError({ e, context, query, res, requestStarted @@ -1990,7 +2018,7 @@ class ApiGateway { query, context, res: (message, opts) => { - if (!Array.isArray(message) && message.error) { + if (!Array.isArray(message) && 'error' in message && message.error) { error = { message, opts }; } else { result = { message, opts }; @@ -2014,7 +2042,18 @@ class ApiGateway { } protected resToResultFn(res: ExpressResponse) { - return (message, { status }: { status?: number } = {}) => (status ? res.status(status).json(message) : res.json(message)); + return async (message, { status }: { status?: number } = {}) => { + if (status) { + res.status(status); + } + + if (message.isWrapper) { + res.set('Content-Type', 'application/json'); + res.send(Buffer.from(await message.getFinalResult())); + } else { + res.json(message); + } + }; } protected parseQueryParam(query): Query | Query[] { diff --git a/packages/cubejs-api-gateway/src/helpers/prepareAnnotation.ts b/packages/cubejs-api-gateway/src/helpers/prepareAnnotation.ts index 268e354279e0e..db0385e7248d4 100644 --- a/packages/cubejs-api-gateway/src/helpers/prepareAnnotation.ts +++ b/packages/cubejs-api-gateway/src/helpers/prepareAnnotation.ts @@ -154,6 +154,7 @@ function prepareAnnotation(metaConfig: MetaConfig[], query: any) { export default prepareAnnotation; export { ConfigItem, + GranularityMeta, annotation, prepareAnnotation, }; diff --git a/packages/cubejs-api-gateway/src/helpers/toConfigMap.ts b/packages/cubejs-api-gateway/src/helpers/toConfigMap.ts index e95cb8aba0786..5766d0763a847 100644 --- a/packages/cubejs-api-gateway/src/helpers/toConfigMap.ts +++ b/packages/cubejs-api-gateway/src/helpers/toConfigMap.ts @@ -28,7 +28,7 @@ type MetaConfigMap = { }; /** - * Convert cpecified array of MetaConfig objects to the + * Convert specified array of MetaConfig objects to the * MetaConfigMap. */ function toConfigMap(metaConfig: MetaConfig[]): MetaConfigMap { diff --git a/packages/cubejs-api-gateway/src/helpers/transformData.ts b/packages/cubejs-api-gateway/src/helpers/transformData.ts deleted file mode 100644 index a9c4f4b9a294e..0000000000000 --- a/packages/cubejs-api-gateway/src/helpers/transformData.ts +++ /dev/null @@ -1,373 +0,0 @@ -/** - * @license Apache-2.0 - * @copyright Cube Dev, Inc. - * @fileoverview - * transformData function and related types definition. - */ - -import R from 'ramda'; -import { UserError } from '../UserError'; -import { ConfigItem } from './prepareAnnotation'; -import { - DBResponsePrimitive, - DBResponseValue, - transformValue, -} from './transformValue'; -import { - NormalizedQuery, - QueryTimeDimension -} from '../types/query'; -import { - ResultType, - QueryType, -} from '../types/strings'; -import { - ResultType as ResultTypeEnum, - QueryType as QueryTypeEnum, -} from '../types/enums'; - -const COMPARE_DATE_RANGE_FIELD = 'compareDateRange'; -const COMPARE_DATE_RANGE_SEPARATOR = ' - '; -const BLENDING_QUERY_KEY_PREFIX = 'time.'; -const BLENDING_QUERY_RES_SEPARATOR = '.'; -const MEMBER_SEPARATOR = '.'; - -/** - * SQL aliases to cube properties hash map. - */ -type AliasToMemberMap = { [alias: string]: string }; - -/** - * Parse date range value from time dimension. - * @internal - */ -function getDateRangeValue( - timeDimensions?: QueryTimeDimension[] -): string { - if (!timeDimensions) { - throw new UserError( - 'QueryTimeDimension should be specified ' + - 'for the compare date range query.' - ); - } else { - const [dim] = timeDimensions; - if (!dim.dateRange) { - throw new UserError( - `${'Inconsistent QueryTimeDimension configuration ' + - 'for the compare date range query, dateRange required: '}${ - dim}` - ); - } else if (typeof dim.dateRange === 'string') { - throw new UserError( - 'Inconsistent dateRange configuration for the ' + - `compare date range query: ${dim.dateRange}` - ); - } else { - return dim.dateRange.join(COMPARE_DATE_RANGE_SEPARATOR); - } - } -} - -/** - * Parse blending query key from time time dimension. - * @internal - */ -function getBlendingQueryKey( - timeDimensions?: QueryTimeDimension[] -): string { - if (!timeDimensions) { - throw new UserError( - 'QueryTimeDimension should be specified ' + - 'for the blending query.' - ); - } else { - const [dim] = timeDimensions; - if (!dim.granularity) { - throw new UserError( - 'Inconsistent QueryTimeDimension configuration ' + - `for the blending query, granularity required: ${dim}` - ); - } else { - return BLENDING_QUERY_KEY_PREFIX + dim.granularity; - } - } -} - -/** - * Parse blending response key from time time dimension. - * @internal - */ -function getBlendingResponseKey( - timeDimensions?: QueryTimeDimension[] -): string { - if (!timeDimensions) { - throw new UserError( - 'QueryTimeDimension should be specified ' + - 'for the blending query.' - ); - } else { - const [dim] = timeDimensions; - if (!dim.granularity) { - throw new UserError( - 'Inconsistent QueryTimeDimension configuration ' + - `for the blending query, granularity required: ${dim}` - ); - } else if (!dim.dimension) { - throw new UserError( - 'Inconsistent QueryTimeDimension configuration ' + - `for the blending query, dimension required: ${dim}` - ); - } else { - return dim.dimension + - BLENDING_QUERY_RES_SEPARATOR + - dim.granularity; - } - } -} - -/** - * Parse members names from request/response. - * @internal - */ -function getMembers( - queryType: QueryType, - query: NormalizedQuery, - dbData: { [sqlAlias: string]: DBResponseValue }[], - aliasToMemberNameMap: AliasToMemberMap, - annotation: { [member: string]: ConfigItem }, -): { [member: string]: string } { - const members: { [member: string]: string } = {}; - if (!dbData.length) { - return members; - } - const columns = Object.keys(dbData[0]); - columns.forEach((column) => { - if (!aliasToMemberNameMap[column] || !annotation[aliasToMemberNameMap[column]]) { - throw new UserError( - `You requested hidden member: '${ - column - }'. Please make it visible using \`shown: true\`. ` + - 'Please note primaryKey fields are `shown: false` by ' + - 'default: https://cube.dev/docs/schema/reference/joins#' + - 'setting-a-primary-key.' - ); - } - members[aliasToMemberNameMap[column]] = column; - const path = aliasToMemberNameMap[column] - .split(MEMBER_SEPARATOR); - const calcMember = - [path[0], path[1]].join(MEMBER_SEPARATOR); - if ( - path.length === 3 && - query.dimensions?.indexOf(calcMember) === -1 - ) { - members[calcMember] = column; - } - }); - if (queryType === QueryTypeEnum.COMPARE_DATE_RANGE_QUERY) { - members[COMPARE_DATE_RANGE_FIELD] = - QueryTypeEnum.COMPARE_DATE_RANGE_QUERY; - } else if (queryType === QueryTypeEnum.BLENDING_QUERY) { - members[getBlendingQueryKey(query.timeDimensions)] = - // @ts-ignore - members[query.timeDimensions[0].dimension]; - } - return members; -} - -/** - * Convert DB response object to the compact output format. - * @internal - * @todo should we use transformValue for blending query? - */ -function getCompactRow( - membersToAliasMap: { [member: string]: string }, - annotation: { [member: string]: ConfigItem }, - queryType: QueryType, - members: string[], - timeDimensions: QueryTimeDimension[] | undefined, - dbRow: { [sqlAlias: string]: DBResponseValue }, -): DBResponsePrimitive[] { - const row: DBResponsePrimitive[] = []; - members.forEach((m: string) => { - if (annotation[m]) { - row.push( - transformValue( - dbRow[membersToAliasMap[m]], - annotation[m].type - ), - ); - } - }); - if (queryType === QueryTypeEnum.COMPARE_DATE_RANGE_QUERY) { - row.push( - getDateRangeValue(timeDimensions) - ); - } else if (queryType === QueryTypeEnum.BLENDING_QUERY) { - row.push( - dbRow[ - membersToAliasMap[ - getBlendingResponseKey(timeDimensions) - ] - ] as DBResponsePrimitive - ); - } - return row; -} - -/** - * Convert DB response object to the vanila output format. - * @todo rewrite me please! - * @internal - */ -function getVanilaRow( - aliasToMemberNameMap: AliasToMemberMap, - annotation: { [member: string]: ConfigItem }, - queryType: QueryType, - query: NormalizedQuery, - dbRow: { [sqlAlias: string]: DBResponseValue }, -): { [member: string]: DBResponsePrimitive } { - const row = R.pipe( - R.toPairs, - R.map(p => { - const memberName = aliasToMemberNameMap[p[0]]; - const annotationForMember = annotation[memberName]; - if (!annotationForMember) { - throw new UserError( - `You requested hidden member: '${ - p[0] - }'. Please make it visible using \`shown: true\`. ` + - 'Please note primaryKey fields are `shown: false` by ' + - 'default: https://cube.dev/docs/schema/reference/joins#' + - 'setting-a-primary-key.' - ); - } - const transformResult = [ - memberName, - transformValue( - p[1] as DBResponseValue, - annotationForMember.type - ) - ]; - const path = memberName.split(MEMBER_SEPARATOR); - - /** - * Time dimensions without granularity. - * @deprecated - * @todo backward compatibility for referencing - */ - const memberNameWithoutGranularity = - [path[0], path[1]].join(MEMBER_SEPARATOR); - if ( - path.length === 3 && - (query.dimensions || []) - .indexOf(memberNameWithoutGranularity) === -1 - ) { - return [ - transformResult, - [ - memberNameWithoutGranularity, - transformResult[1] - ] - ]; - } - - return [transformResult]; - }), - // @ts-ignore - R.unnest, - R.fromPairs - // @ts-ignore - )(dbRow); - if (queryType === QueryTypeEnum.COMPARE_DATE_RANGE_QUERY) { - return { - ...row, - compareDateRange: getDateRangeValue(query.timeDimensions) - }; - } else if (queryType === QueryTypeEnum.BLENDING_QUERY) { - return { - ...row, - [getBlendingQueryKey(query.timeDimensions)]: - row[getBlendingResponseKey(query.timeDimensions)] - }; - } - return row as { [member: string]: DBResponsePrimitive; }; -} - -/** - * Transforms queried data array to the output format. - */ -function transformData( - aliasToMemberNameMap: AliasToMemberMap, - annotation: { [member: string]: ConfigItem }, - data: { [sqlAlias: string]: unknown }[], - query: NormalizedQuery, - queryType: QueryType, - resType?: ResultType -): { - members: string[], - dataset: DBResponsePrimitive[][] -} | { - [member: string]: DBResponsePrimitive -}[] { - const d = data as { [sqlAlias: string]: DBResponseValue }[]; - const membersToAliasMap = getMembers( - queryType, - query, - d, - aliasToMemberNameMap, - annotation, - ); - const members: string[] = Object.keys(membersToAliasMap); - const dataset: DBResponsePrimitive[][] | { - [member: string]: DBResponsePrimitive - }[] = d.map((r) => { - const row: DBResponsePrimitive[] | { - [member: string]: DBResponsePrimitive - } = resType === ResultTypeEnum.COMPACT - ? getCompactRow( - membersToAliasMap, - annotation, - queryType, - members, - query.timeDimensions, - r, - ) - : getVanilaRow( - aliasToMemberNameMap, - annotation, - queryType, - query, - r, - ); - return row; - }) as DBResponsePrimitive[][] | { - [member: string]: DBResponsePrimitive - }[]; - return (resType === ResultTypeEnum.COMPACT - ? { members, dataset } - : dataset - ) as { - members: string[], - dataset: DBResponsePrimitive[][] - } | { - [member: string]: DBResponsePrimitive - }[]; -} - -export default transformData; -export { - AliasToMemberMap, - COMPARE_DATE_RANGE_FIELD, - COMPARE_DATE_RANGE_SEPARATOR, - BLENDING_QUERY_KEY_PREFIX, - BLENDING_QUERY_RES_SEPARATOR, - MEMBER_SEPARATOR, - getDateRangeValue, - getBlendingQueryKey, - getBlendingResponseKey, - getMembers, - getCompactRow, - getVanilaRow, - transformData, -}; diff --git a/packages/cubejs-api-gateway/src/helpers/transformValue.ts b/packages/cubejs-api-gateway/src/helpers/transformValue.ts deleted file mode 100644 index 78d80f16f15bc..0000000000000 --- a/packages/cubejs-api-gateway/src/helpers/transformValue.ts +++ /dev/null @@ -1,48 +0,0 @@ -/** - * @license Apache-2.0 - * @copyright Cube Dev, Inc. - * @fileoverview - * transformValue function and related types definition. - */ - -import moment, { MomentInput } from 'moment'; - -/** - * Query 'or'-filters type definition. - */ -type DBResponsePrimitive = - null | - boolean | - number | - string; - -type DBResponseValue = - Date | - DBResponsePrimitive | - { value: DBResponsePrimitive }; - -/** - * Transform cpecified `value` with specified `type` to the network - * protocol type. - */ -function transformValue( - value: DBResponseValue, - type: string -): DBResponsePrimitive { - // TODO: support for max time - if (value && (type === 'time' || value instanceof Date)) { - return ( - value instanceof Date - ? moment(value) - : moment.utc(value as MomentInput) - ).format(moment.HTML5_FMT.DATETIME_LOCAL_MS); - } - return value as DBResponsePrimitive; -} - -export default transformValue; -export { - DBResponsePrimitive, - DBResponseValue, - transformValue, -}; diff --git a/packages/cubejs-api-gateway/src/index.ts b/packages/cubejs-api-gateway/src/index.ts index ced6087014c9c..a990e663bf6e8 100644 --- a/packages/cubejs-api-gateway/src/index.ts +++ b/packages/cubejs-api-gateway/src/index.ts @@ -4,3 +4,4 @@ export * from './interfaces'; export * from './CubejsHandlerError'; export * from './UserError'; export { getRequestIdFromRequest } from './requestParser'; +export { TransformDataRequest } from './types/responses'; diff --git a/packages/cubejs-api-gateway/src/interfaces.ts b/packages/cubejs-api-gateway/src/interfaces.ts index aad8f106e3e85..681fa4507228c 100644 --- a/packages/cubejs-api-gateway/src/interfaces.ts +++ b/packages/cubejs-api-gateway/src/interfaces.ts @@ -10,6 +10,7 @@ import { import { QueryType, + ResultType, } from './types/enums'; import { @@ -43,9 +44,24 @@ import { QueryRequest } from './types/request'; +import { + AliasToMemberMap, + TransformDataResponse +} from './types/responses'; + +import { + ConfigItem, + GranularityMeta +} from './helpers/prepareAnnotation'; + export { + AliasToMemberMap, + TransformDataResponse, + ConfigItem, + GranularityMeta, QueryTimeDimensionGranularity, QueryType, + ResultType, QueryFilter, LogicalAndFilter, LogicalOrFilter, diff --git a/packages/cubejs-api-gateway/src/types/request.ts b/packages/cubejs-api-gateway/src/types/request.ts index ed6c5bbd80174..3393554fd8d65 100644 --- a/packages/cubejs-api-gateway/src/types/request.ts +++ b/packages/cubejs-api-gateway/src/types/request.ts @@ -6,6 +6,7 @@ */ import type { Request as ExpressRequest } from 'express'; +import type { DataResult } from '@cubejs-backend/native'; import { RequestType, ApiType, ResultType } from './strings'; import { Query } from './query'; @@ -105,7 +106,7 @@ type MetaResponseResultFn = (message: MetaResponse | ErrorResponse) => void; */ type ResponseResultFn = ( - message: (Record | Record[]) | ErrorResponse, + message: (Record | Record[]) | DataResult | ErrorResponse, extra?: { status: number } ) => void; diff --git a/packages/cubejs-api-gateway/src/types/responses.ts b/packages/cubejs-api-gateway/src/types/responses.ts new file mode 100644 index 0000000000000..d5e1d8ea1e0aa --- /dev/null +++ b/packages/cubejs-api-gateway/src/types/responses.ts @@ -0,0 +1,35 @@ +import type { ConfigItem } from '../helpers/prepareAnnotation'; +import type { NormalizedQuery } from './query'; +import type { QueryType, ResultType } from './strings'; + +export type DBResponsePrimitive = + null | + boolean | + number | + string; + +export type DBResponseValue = + Date | + DBResponsePrimitive | + { value: DBResponsePrimitive }; + +export type TransformDataResponse = { + members: string[], + dataset: DBResponsePrimitive[][] +} | { + [member: string]: DBResponsePrimitive +}[]; + +/** + * SQL aliases to cube properties hash map. + */ +export type AliasToMemberMap = { [alias: string]: string }; + +export type TransformDataRequest = { + aliasToMemberNameMap: { [alias: string]: string }, + annotation: { [member: string]: ConfigItem }, + data: { [sqlAlias: string]: unknown }[], + query: NormalizedQuery, + queryType: QueryType, + resType?: ResultType +}; diff --git a/packages/cubejs-api-gateway/test/helpers/transformData.test.ts b/packages/cubejs-api-gateway/test/helpers/transformData.test.ts deleted file mode 100644 index 8936bd3d44418..0000000000000 --- a/packages/cubejs-api-gateway/test/helpers/transformData.test.ts +++ /dev/null @@ -1,1339 +0,0 @@ -/** - * @license Apache-2.0 - * @copyright Cube Dev, Inc. - * @fileoverview transformData related helpers unit tests. - */ - -/* globals describe,test,expect */ -/* eslint-disable import/no-duplicates */ -/* eslint-disable @typescript-eslint/no-duplicate-imports */ - -import { - QueryTimeDimension, - NormalizedQuery, -} from '../../src/types/query'; -import { - QueryType as QueryTypeEnum, - ResultType as ResultTypeEnum, -} from '../../src/types/enums'; -import { - DBResponseValue, -} from '../../src/helpers/transformValue'; -import { - ConfigItem, -} from '../../src/helpers/prepareAnnotation'; -import transformDataDefault - from '../../src/helpers/transformData'; -import { - COMPARE_DATE_RANGE_FIELD, - COMPARE_DATE_RANGE_SEPARATOR, - BLENDING_QUERY_KEY_PREFIX, - BLENDING_QUERY_RES_SEPARATOR, - MEMBER_SEPARATOR, - getDateRangeValue, - getBlendingQueryKey, - getBlendingResponseKey, - getMembers, - getCompactRow, - getVanilaRow, - transformData, -} from '../../src/helpers/transformData'; -import { - QueryType, -} from '../../src/types/strings'; - -const mockData = { - regular_discount_by_city: { - query: { - dimensions: [ - 'ECommerceRecordsUs2021.city' - ], - measures: [ - 'ECommerceRecordsUs2021.avg_discount' - ], - limit: 2 - }, - data: { - aliasToMemberNameMap: { - e_commerce_records_us2021__avg_discount: 'ECommerceRecordsUs2021.avg_discount', - e_commerce_records_us2021__city: 'ECommerceRecordsUs2021.city' - }, - annotation: { - 'ECommerceRecordsUs2021.avg_discount': { - title: 'E Commerce Records Us2021 Avg Discount', - shortTitle: 'Avg Discount', - type: 'number', - drillMembers: [], - drillMembersGrouped: { - measures: [], - dimensions: [] - } - }, - 'ECommerceRecordsUs2021.city': { - title: 'E Commerce Records Us2021 City', - shortTitle: 'City', - type: 'string' - } - }, - data: [ - { - e_commerce_records_us2021__city: 'Missouri City', - e_commerce_records_us2021__avg_discount: '0.80000000000000000000' - }, - { - e_commerce_records_us2021__city: 'Abilene', - e_commerce_records_us2021__avg_discount: '0.80000000000000000000' - } - ], - query: { - dimensions: [ - 'ECommerceRecordsUs2021.city' - ], - measures: [ - 'ECommerceRecordsUs2021.avg_discount' - ], - limit: 2, - rowLimit: 2, - timezone: 'UTC', - order: [], - filters: [], - timeDimensions: [] - }, - queryType: 'regularQuery', - result_default: [ - { - 'ECommerceRecordsUs2021.city': 'Missouri City', - 'ECommerceRecordsUs2021.avg_discount': '0.80000000000000000000' - }, - { - 'ECommerceRecordsUs2021.city': 'Abilene', - 'ECommerceRecordsUs2021.avg_discount': '0.80000000000000000000' - } - ], - result_compact: { - members: ['ECommerceRecordsUs2021.city', 'ECommerceRecordsUs2021.avg_discount'], - dataset: [['Missouri City', '0.80000000000000000000'], ['Abilene', '0.80000000000000000000']], - } - } - }, - regular_profit_by_postal_code: { - query: { - dimensions: [ - 'ECommerceRecordsUs2021.postalCode' - ], - measures: [ - 'ECommerceRecordsUs2021.avg_profit' - ], - limit: 2 - }, - data: { - aliasToMemberNameMap: { - e_commerce_records_us2021__avg_profit: 'ECommerceRecordsUs2021.avg_profit', - e_commerce_records_us2021__postal_code: 'ECommerceRecordsUs2021.postalCode' - }, - annotation: { - 'ECommerceRecordsUs2021.avg_profit': { - title: 'E Commerce Records Us2021 Avg Profit', - shortTitle: 'Avg Profit', - type: 'number', - drillMembers: [], - drillMembersGrouped: { - measures: [], - dimensions: [] - } - }, - 'ECommerceRecordsUs2021.postalCode': { - title: 'E Commerce Records Us2021 Postal Code', - shortTitle: 'Postal Code', - type: 'string' - } - }, - data: [ - { - e_commerce_records_us2021__postal_code: '95823', - e_commerce_records_us2021__avg_profit: '646.1258666666666667' - }, - { - e_commerce_records_us2021__postal_code: '64055', - e_commerce_records_us2021__avg_profit: '487.8315000000000000' - } - ], - query: { - dimensions: [ - 'ECommerceRecordsUs2021.postalCode' - ], - measures: [ - 'ECommerceRecordsUs2021.avg_profit' - ], - limit: 2, - rowLimit: 2, - timezone: 'UTC', - order: [], - filters: [], - timeDimensions: [] - }, - queryType: 'regularQuery', - result_default: [ - { - 'ECommerceRecordsUs2021.postalCode': '95823', - 'ECommerceRecordsUs2021.avg_profit': '646.1258666666666667' - }, - { - 'ECommerceRecordsUs2021.postalCode': '64055', - 'ECommerceRecordsUs2021.avg_profit': '487.8315000000000000' - } - ], - result_compact: { - members: [ - 'ECommerceRecordsUs2021.postalCode', - 'ECommerceRecordsUs2021.avg_profit', - ], - dataset: [ - ['95823', '646.1258666666666667'], - ['64055', '487.8315000000000000'] - ], - } - } - }, - compare_date_range_count_by_order_date: { - http_params: { - queryType: 'whatever value or nothing, \'multi\' to apply pivot transformation' - }, - query: { - measures: ['ECommerceRecordsUs2021.count'], - timeDimensions: [ - { - dimension: 'ECommerceRecordsUs2021.orderDate', - granularity: 'day', - compareDateRange: [ - ['2020-01-01', '2020-01-31'], - ['2020-03-01', '2020-03-31'] - ] - } - ], - limit: 2 - }, - data: [{ - aliasToMemberNameMap: { - e_commerce_records_us2021__count: 'ECommerceRecordsUs2021.count', - e_commerce_records_us2021__order_date_day: 'ECommerceRecordsUs2021.orderDate.day' - }, - annotation: { - 'ECommerceRecordsUs2021.count': { - title: 'E Commerce Records Us2021 Count', - shortTitle: 'Count', - type: 'number', - drillMembers: [ - 'ECommerceRecordsUs2021.city', - 'ECommerceRecordsUs2021.country', - 'ECommerceRecordsUs2021.customerId', - 'ECommerceRecordsUs2021.orderId', - 'ECommerceRecordsUs2021.productId', - 'ECommerceRecordsUs2021.productName', - 'ECommerceRecordsUs2021.orderDate' - ], - drillMembersGrouped: { - measures: [], - dimensions: [ - 'ECommerceRecordsUs2021.city', - 'ECommerceRecordsUs2021.country', - 'ECommerceRecordsUs2021.customerId', - 'ECommerceRecordsUs2021.orderId', - 'ECommerceRecordsUs2021.productId', - 'ECommerceRecordsUs2021.productName', - 'ECommerceRecordsUs2021.orderDate' - ] - } - }, - 'ECommerceRecordsUs2021.orderDate.day': { - title: 'E Commerce Records Us2021 Order Date', - shortTitle: 'Order Date', - type: 'time' - }, - 'ECommerceRecordsUs2021.orderDate': { - title: 'E Commerce Records Us2021 Order Date', - shortTitle: 'Order Date', - type: 'time' - } - }, - data: [ - { - e_commerce_records_us2021__order_date_day: '2020-01-01T00:00:00.000', - e_commerce_records_us2021__count: '10' - }, - { - e_commerce_records_us2021__order_date_day: '2020-01-02T00:00:00.000', - e_commerce_records_us2021__count: '8' - } - ], - query: { - measures: [ - 'ECommerceRecordsUs2021.count' - ], - timeDimensions: [ - { - dimension: 'ECommerceRecordsUs2021.orderDate', - granularity: 'day', - dateRange: [ - '2020-01-01T00:00:00.000', - '2020-01-31T23:59:59.999' - ] - } - ], - limit: 2, - rowLimit: 2, - timezone: 'UTC', - order: [], - filters: [], - dimensions: [] - }, - queryType: 'compareDateRangeQuery', - result_default: [ - { - 'ECommerceRecordsUs2021.orderDate.day': '2020-01-01T00:00:00.000', - 'ECommerceRecordsUs2021.orderDate': '2020-01-01T00:00:00.000', - 'ECommerceRecordsUs2021.count': '10', - compareDateRange: '2020-01-01T00:00:00.000 - 2020-01-31T23:59:59.999' - }, - { - 'ECommerceRecordsUs2021.orderDate.day': '2020-01-02T00:00:00.000', - 'ECommerceRecordsUs2021.orderDate': '2020-01-02T00:00:00.000', - 'ECommerceRecordsUs2021.count': '8', - compareDateRange: '2020-01-01T00:00:00.000 - 2020-01-31T23:59:59.999' - } - ], - result_compact: { - members: [ - 'ECommerceRecordsUs2021.orderDate.day', - 'ECommerceRecordsUs2021.orderDate', - 'ECommerceRecordsUs2021.count', - 'compareDateRange', - ], - dataset: [ - [ - '2020-01-01T00:00:00.000', - '2020-01-01T00:00:00.000', - '10', - '2020-01-01T00:00:00.000 - 2020-01-31T23:59:59.999', - ], - [ - '2020-01-02T00:00:00.000', - '2020-01-02T00:00:00.000', - '8', - '2020-01-01T00:00:00.000 - 2020-01-31T23:59:59.999' - ], - ], - }, - }, { - aliasToMemberNameMap: { - e_commerce_records_us2021__count: 'ECommerceRecordsUs2021.count', - e_commerce_records_us2021__order_date_day: 'ECommerceRecordsUs2021.orderDate.day' - }, - annotation: { - 'ECommerceRecordsUs2021.count': { - title: 'E Commerce Records Us2021 Count', - shortTitle: 'Count', - type: 'number', - drillMembers: [ - 'ECommerceRecordsUs2021.city', - 'ECommerceRecordsUs2021.country', - 'ECommerceRecordsUs2021.customerId', - 'ECommerceRecordsUs2021.orderId', - 'ECommerceRecordsUs2021.productId', - 'ECommerceRecordsUs2021.productName', - 'ECommerceRecordsUs2021.orderDate' - ], - drillMembersGrouped: { - measures: [], - dimensions: [ - 'ECommerceRecordsUs2021.city', - 'ECommerceRecordsUs2021.country', - 'ECommerceRecordsUs2021.customerId', - 'ECommerceRecordsUs2021.orderId', - 'ECommerceRecordsUs2021.productId', - 'ECommerceRecordsUs2021.productName', - 'ECommerceRecordsUs2021.orderDate' - ] - } - }, - 'ECommerceRecordsUs2021.orderDate.day': { - title: 'E Commerce Records Us2021 Order Date', - shortTitle: 'Order Date', - type: 'time' - }, - 'ECommerceRecordsUs2021.orderDate': { - title: 'E Commerce Records Us2021 Order Date', - shortTitle: 'Order Date', - type: 'time' - } - }, - data: [ - { - e_commerce_records_us2021__order_date_day: '2020-03-02T00:00:00.000', - e_commerce_records_us2021__count: '11' - }, - { - e_commerce_records_us2021__order_date_day: '2020-03-03T00:00:00.000', - e_commerce_records_us2021__count: '7' - } - ], - query: { - measures: [ - 'ECommerceRecordsUs2021.count' - ], - timeDimensions: [ - { - dimension: 'ECommerceRecordsUs2021.orderDate', - granularity: 'day', - dateRange: [ - '2020-03-01T00:00:00.000', - '2020-03-31T23:59:59.999' - ] - } - ], - limit: 2, - rowLimit: 2, - timezone: 'UTC', - order: [], - filters: [], - dimensions: [] - }, - queryType: 'compareDateRangeQuery', - result_default: [ - { - 'ECommerceRecordsUs2021.orderDate.day': '2020-03-02T00:00:00.000', - 'ECommerceRecordsUs2021.orderDate': '2020-03-02T00:00:00.000', - 'ECommerceRecordsUs2021.count': '11', - compareDateRange: '2020-03-01T00:00:00.000 - 2020-03-31T23:59:59.999' - }, - { - 'ECommerceRecordsUs2021.orderDate.day': '2020-03-03T00:00:00.000', - 'ECommerceRecordsUs2021.orderDate': '2020-03-03T00:00:00.000', - 'ECommerceRecordsUs2021.count': '7', - compareDateRange: '2020-03-01T00:00:00.000 - 2020-03-31T23:59:59.999' - } - ], - result_compact: { - members: [ - 'ECommerceRecordsUs2021.orderDate.day', - 'ECommerceRecordsUs2021.orderDate', - 'ECommerceRecordsUs2021.count', - 'compareDateRange', - ], - dataset: [ - [ - '2020-03-02T00:00:00.000', - '2020-03-02T00:00:00.000', - '11', - '2020-03-01T00:00:00.000 - 2020-03-31T23:59:59.999', - ], - [ - '2020-03-03T00:00:00.000', - '2020-03-03T00:00:00.000', - '7', - '2020-03-01T00:00:00.000 - 2020-03-31T23:59:59.999' - ], - ], - }, - }] - }, - blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode: { - http_params: { - queryType: 'whatever value or nothing, \'multi\' to apply pivot transformation' - }, - query: [{ - measures: ['ECommerceRecordsUs2021.avg_discount'], - timeDimensions: [ - { - dimension: 'ECommerceRecordsUs2021.orderDate', - granularity: 'month', - dateRange: ['2020-01-01', '2020-12-30'] - } - ], - filters: [{ - member: 'ECommerceRecordsUs2021.shipMode', - operator: 'equals', - values: ['Standard Class'] - }], - limit: 2 - }, { - measures: ['ECommerceRecordsUs2021.avg_discount'], - timeDimensions: [ - { - dimension: 'ECommerceRecordsUs2021.orderDate', - granularity: 'month', - dateRange: ['2020-01-01', '2020-12-30'] - } - ], - filters: [{ - member: 'ECommerceRecordsUs2021.shipMode', - operator: 'equals', - values: ['First Class'] - }], - limit: 2 - }], - data: [{ - aliasToMemberNameMap: { - e_commerce_records_us2021__avg_discount: 'ECommerceRecordsUs2021.avg_discount', - e_commerce_records_us2021__order_date_month: 'ECommerceRecordsUs2021.orderDate.month' - }, - annotation: { - 'ECommerceRecordsUs2021.avg_discount': { - title: 'E Commerce Records Us2021 Avg Discount', - shortTitle: 'Avg Discount', - type: 'number', - drillMembers: [], - drillMembersGrouped: { - measures: [], - dimensions: [] - } - }, - 'ECommerceRecordsUs2021.orderDate.month': { - title: 'E Commerce Records Us2021 Order Date', - shortTitle: 'Order Date', - type: 'time' - }, - 'ECommerceRecordsUs2021.orderDate': { - title: 'E Commerce Records Us2021 Order Date', - shortTitle: 'Order Date', - type: 'time' - } - }, - data: [ - { - e_commerce_records_us2021__order_date_month: '2020-01-01T00:00:00.000', - e_commerce_records_us2021__avg_discount: '0.15638297872340425532' - }, - { - e_commerce_records_us2021__order_date_month: '2020-02-01T00:00:00.000', - e_commerce_records_us2021__avg_discount: '0.17573529411764705882' - } - ], - query: { - measures: [ - 'ECommerceRecordsUs2021.avg_discount' - ], - timeDimensions: [ - { - dimension: 'ECommerceRecordsUs2021.orderDate', - granularity: 'month', - dateRange: [ - '2020-01-01T00:00:00.000', - '2020-12-30T23:59:59.999' - ] - } - ], - filters: [ - { - operator: 'equals', - values: [ - 'Standard Class' - ], - member: 'ECommerceRecordsUs2021.shipMode' - } - ], - limit: 2, - rowLimit: 2, - timezone: 'UTC', - order: [], - dimensions: [] - }, - queryType: 'blendingQuery', - result_default: [ - { - 'ECommerceRecordsUs2021.orderDate.month': '2020-01-01T00:00:00.000', - 'ECommerceRecordsUs2021.orderDate': '2020-01-01T00:00:00.000', - 'ECommerceRecordsUs2021.avg_discount': '0.15638297872340425532', - 'time.month': '2020-01-01T00:00:00.000' - }, - { - 'ECommerceRecordsUs2021.orderDate.month': '2020-02-01T00:00:00.000', - 'ECommerceRecordsUs2021.orderDate': '2020-02-01T00:00:00.000', - 'ECommerceRecordsUs2021.avg_discount': '0.17573529411764705882', - 'time.month': '2020-02-01T00:00:00.000' - } - ], - result_compact: { - members: [ - 'ECommerceRecordsUs2021.orderDate.month', - 'ECommerceRecordsUs2021.orderDate', - 'ECommerceRecordsUs2021.avg_discount', - 'time.month', - ], - dataset: [ - [ - '2020-01-01T00:00:00.000', - '2020-01-01T00:00:00.000', - '0.15638297872340425532', - '2020-01-01T00:00:00.000', - ], - [ - '2020-02-01T00:00:00.000', - '2020-02-01T00:00:00.000', - '0.17573529411764705882', - '2020-02-01T00:00:00.000', - ], - ], - }, - }, { - aliasToMemberNameMap: { - e_commerce_records_us2021__avg_discount: 'ECommerceRecordsUs2021.avg_discount', - e_commerce_records_us2021__order_date_month: 'ECommerceRecordsUs2021.orderDate.month' - }, - annotation: { - 'ECommerceRecordsUs2021.avg_discount': { - title: 'E Commerce Records Us2021 Avg Discount', - shortTitle: 'Avg Discount', - type: 'number', - drillMembers: [], - drillMembersGrouped: { - measures: [], - dimensions: [] - } - }, - 'ECommerceRecordsUs2021.orderDate.month': { - title: 'E Commerce Records Us2021 Order Date', - shortTitle: 'Order Date', - type: 'time' - }, - 'ECommerceRecordsUs2021.orderDate': { - title: 'E Commerce Records Us2021 Order Date', - shortTitle: 'Order Date', - type: 'time' - } - }, - data: [ - { - e_commerce_records_us2021__order_date_month: '2020-01-01T00:00:00.000', - e_commerce_records_us2021__avg_discount: '0.28571428571428571429' - }, - { - e_commerce_records_us2021__order_date_month: '2020-02-01T00:00:00.000', - e_commerce_records_us2021__avg_discount: '0.21777777777777777778' - } - ], - query: { - measures: [ - 'ECommerceRecordsUs2021.avg_discount' - ], - timeDimensions: [ - { - dimension: 'ECommerceRecordsUs2021.orderDate', - granularity: 'month', - dateRange: [ - '2020-01-01T00:00:00.000', - '2020-12-30T23:59:59.999' - ] - } - ], - filters: [ - { - operator: 'equals', - values: [ - 'First Class' - ], - member: 'ECommerceRecordsUs2021.shipMode' - } - ], - limit: 2, - rowLimit: 2, - timezone: 'UTC', - order: [], - dimensions: [] - }, - queryType: 'blendingQuery', - result_default: [{ - 'ECommerceRecordsUs2021.orderDate.month': '2020-01-01T00:00:00.000', - 'ECommerceRecordsUs2021.orderDate': '2020-01-01T00:00:00.000', - 'ECommerceRecordsUs2021.avg_discount': '0.28571428571428571429', - 'time.month': '2020-01-01T00:00:00.000' - }, - { - 'ECommerceRecordsUs2021.orderDate.month': '2020-02-01T00:00:00.000', - 'ECommerceRecordsUs2021.orderDate': '2020-02-01T00:00:00.000', - 'ECommerceRecordsUs2021.avg_discount': '0.21777777777777777778', - 'time.month': '2020-02-01T00:00:00.000' - }], - result_compact: { - members: [ - 'ECommerceRecordsUs2021.orderDate.month', - 'ECommerceRecordsUs2021.orderDate', - 'ECommerceRecordsUs2021.avg_discount', - 'time.month', - ], - dataset: [ - [ - '2020-01-01T00:00:00.000', - '2020-01-01T00:00:00.000', - '0.28571428571428571429', - '2020-01-01T00:00:00.000', - ], - [ - '2020-02-01T00:00:00.000', - '2020-02-01T00:00:00.000', - '0.21777777777777777778', - '2020-02-01T00:00:00.000', - ], - ], - }, - }] - } -}; - -describe('transformData helpers', () => { - test('export looks as expected', () => { - expect(transformDataDefault).toBeDefined(); - expect(COMPARE_DATE_RANGE_FIELD).toBeDefined(); - expect(COMPARE_DATE_RANGE_SEPARATOR).toBeDefined(); - expect(BLENDING_QUERY_KEY_PREFIX).toBeDefined(); - expect(BLENDING_QUERY_RES_SEPARATOR).toBeDefined(); - expect(MEMBER_SEPARATOR).toBeDefined(); - expect(getDateRangeValue).toBeDefined(); - expect(getBlendingQueryKey).toBeDefined(); - expect(getBlendingResponseKey).toBeDefined(); - expect(getMembers).toBeDefined(); - expect(getCompactRow).toBeDefined(); - expect(getVanilaRow).toBeDefined(); - expect(transformData).toBeDefined(); - expect(transformData).toEqual(transformDataDefault); - }); - - test('getDateRangeValue helper', () => { - const timeDimensions = JSON.parse( - JSON.stringify( - mockData - .blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode - .data[0] - .query - .timeDimensions - ) - ) as QueryTimeDimension[]; - - expect(() => { getDateRangeValue(); }).toThrow( - 'QueryTimeDimension should be specified ' + - 'for the compare date range query.' - ); - - expect(() => { - getDateRangeValue([ - { prop: 'val' } as unknown as QueryTimeDimension - ]); - }).toThrow( - `${'Inconsistent QueryTimeDimension configuration ' + - 'for the compare date range query, dateRange required: '}${ - ({ prop: 'val' }).toString()}` - ); - - expect(() => { - getDateRangeValue([ - { dateRange: 'val' } as unknown as QueryTimeDimension - ]); - }).toThrow( - 'Inconsistent dateRange configuration for the ' + - 'compare date range query: val' - ); - - expect(getDateRangeValue(timeDimensions)).toEqual( - `${ - // @ts-ignore - timeDimensions[0].dateRange[0] - }${ - COMPARE_DATE_RANGE_SEPARATOR - }${ - // @ts-ignore - timeDimensions[0].dateRange[1] - }` - ); - }); - - test('getBlendingQueryKey helper', () => { - const timeDimensions = JSON.parse( - JSON.stringify( - mockData - .blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode - .data[0] - .query - .timeDimensions - ) - ) as QueryTimeDimension[]; - - expect(() => { - getBlendingQueryKey(); - }).toThrow( - 'QueryTimeDimension should be specified ' + - 'for the blending query.' - ); - - expect(() => { - getBlendingQueryKey([ - { prop: 'val' } as unknown as QueryTimeDimension - ]); - }).toThrow( - 'Inconsistent QueryTimeDimension configuration ' + - `for the blending query, granularity required: ${ - ({ prop: 'val' }).toString()}` - ); - - expect(getBlendingQueryKey(timeDimensions)) - .toEqual(`${ - BLENDING_QUERY_KEY_PREFIX - }${ - timeDimensions[0].granularity - }`); - }); - - test('getBlendingResponseKey helper', () => { - const timeDimensions = JSON.parse( - JSON.stringify( - mockData - .blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode - .data[0] - .query - .timeDimensions - ) - ) as QueryTimeDimension[]; - - expect(() => { - getBlendingResponseKey(); - }).toThrow( - 'QueryTimeDimension should be specified ' + - 'for the blending query.' - ); - - expect(() => { - getBlendingResponseKey([ - { prop: 'val' } as unknown as QueryTimeDimension - ]); - }).toThrow( - 'Inconsistent QueryTimeDimension configuration ' + - `for the blending query, granularity required: ${ - ({ prop: 'val' }).toString()}` - ); - - expect(() => { - getBlendingResponseKey([ - { granularity: 'day' } as unknown as QueryTimeDimension - ]); - }).toThrow( - 'Inconsistent QueryTimeDimension configuration ' + - `for the blending query, dimension required: ${ - ({ granularity: 'day' }).toString()}` - ); - - expect(getBlendingResponseKey(timeDimensions)) - .toEqual(`${ - timeDimensions[0].dimension - }${ - BLENDING_QUERY_RES_SEPARATOR - }${ - timeDimensions[0].granularity - }`); - }); - - test('getMembers helper', () => { - let data; - - // throw - data = JSON.parse( - JSON.stringify(mockData.regular_profit_by_postal_code.data) - ); - data.aliasToMemberNameMap = {}; - expect(() => { - getMembers( - data.queryType as QueryTypeEnum, - data.query as unknown as NormalizedQuery, - data.data as { [sqlAlias: string]: DBResponseValue }[], - data.aliasToMemberNameMap, - data.annotation, - ); - }).toThrow( - 'You requested hidden member: \'e_commerce_records_us2021__postal_code\'. ' + - 'Please make it visible using `shown: true`. Please note primaryKey fields are ' + - '`shown: false` by default: ' + - 'https://cube.dev/docs/schema/reference/joins#setting-a-primary-key.' - ); - - // regular - data = JSON.parse( - JSON.stringify(mockData.regular_profit_by_postal_code.data) - ); - data.data = []; - expect(getMembers( - data.queryType as QueryTypeEnum, - data.query as unknown as NormalizedQuery, - data.data as { [sqlAlias: string]: DBResponseValue }[], - data.aliasToMemberNameMap, - data.annotation, - )).toEqual({}); - - data = JSON.parse( - JSON.stringify(mockData.regular_profit_by_postal_code.data) - ); - expect(getMembers( - data.queryType as QueryTypeEnum, - data.query as unknown as NormalizedQuery, - data.data as { [sqlAlias: string]: DBResponseValue }[], - data.aliasToMemberNameMap, - data.annotation, - )).toEqual({ - 'ECommerceRecordsUs2021.postalCode': 'e_commerce_records_us2021__postal_code', - 'ECommerceRecordsUs2021.avg_profit': 'e_commerce_records_us2021__avg_profit' - }); - - // compare date range - data = JSON.parse( - JSON.stringify(mockData.compare_date_range_count_by_order_date.data[0]) - ); - data.data = []; - expect(getMembers( - data.queryType as QueryTypeEnum, - data.query as unknown as NormalizedQuery, - data.data as { [sqlAlias: string]: DBResponseValue }[], - data.aliasToMemberNameMap, - data.annotation, - )).toEqual({}); - - data = JSON.parse( - JSON.stringify(mockData.compare_date_range_count_by_order_date.data[0]) - ); - expect(getMembers( - data.queryType as QueryTypeEnum, - data.query as unknown as NormalizedQuery, - data.data as { [sqlAlias: string]: DBResponseValue }[], - data.aliasToMemberNameMap, - data.annotation, - )).toEqual({ - 'ECommerceRecordsUs2021.orderDate.day': 'e_commerce_records_us2021__order_date_day', - 'ECommerceRecordsUs2021.orderDate': 'e_commerce_records_us2021__order_date_day', - 'ECommerceRecordsUs2021.count': 'e_commerce_records_us2021__count', - compareDateRange: 'compareDateRangeQuery', - }); - - data = JSON.parse( - JSON.stringify(mockData.compare_date_range_count_by_order_date.data[1]) - ); - expect(getMembers( - data.queryType as QueryTypeEnum, - data.query as unknown as NormalizedQuery, - data.data as { [sqlAlias: string]: DBResponseValue }[], - data.aliasToMemberNameMap, - data.annotation, - )).toEqual({ - 'ECommerceRecordsUs2021.orderDate.day': 'e_commerce_records_us2021__order_date_day', - 'ECommerceRecordsUs2021.orderDate': 'e_commerce_records_us2021__order_date_day', - 'ECommerceRecordsUs2021.count': 'e_commerce_records_us2021__count', - compareDateRange: 'compareDateRangeQuery', - }); - - // blending - data = JSON.parse( - JSON.stringify( - mockData - .blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode - .data[0] - ) - ); - data.data = []; - expect(getMembers( - data.queryType as QueryTypeEnum, - data.query as unknown as NormalizedQuery, - data.data as { [sqlAlias: string]: DBResponseValue }[], - data.aliasToMemberNameMap, - data.annotation, - )).toEqual({}); - - data = JSON.parse( - JSON.stringify( - mockData - .blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode - .data[0] - ) - ); - expect(getMembers( - data.queryType as QueryTypeEnum, - data.query as unknown as NormalizedQuery, - data.data as { [sqlAlias: string]: DBResponseValue }[], - data.aliasToMemberNameMap, - data.annotation, - )).toEqual({ - 'ECommerceRecordsUs2021.orderDate.month': 'e_commerce_records_us2021__order_date_month', - 'ECommerceRecordsUs2021.orderDate': 'e_commerce_records_us2021__order_date_month', - 'ECommerceRecordsUs2021.avg_discount': 'e_commerce_records_us2021__avg_discount', - 'time.month': 'e_commerce_records_us2021__order_date_month', - }); - - data = JSON.parse( - JSON.stringify( - mockData - .blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode - .data[1] - ) - ); - expect(getMembers( - data.queryType as QueryTypeEnum, - data.query as unknown as NormalizedQuery, - data.data as { [sqlAlias: string]: DBResponseValue }[], - data.aliasToMemberNameMap, - data.annotation, - )).toEqual({ - 'ECommerceRecordsUs2021.orderDate.month': 'e_commerce_records_us2021__order_date_month', - 'ECommerceRecordsUs2021.orderDate': 'e_commerce_records_us2021__order_date_month', - 'ECommerceRecordsUs2021.avg_discount': 'e_commerce_records_us2021__avg_discount', - 'time.month': 'e_commerce_records_us2021__order_date_month', - }); - }); - - test('getCompactRow helper', () => { - let data; - let membersMap; - let members; - - // regular - data = JSON.parse( - JSON.stringify(mockData.regular_profit_by_postal_code.data) - ); - membersMap = getMembers( - data.queryType as QueryTypeEnum, - data.query as unknown as NormalizedQuery, - data.data as { [sqlAlias: string]: DBResponseValue }[], - data.aliasToMemberNameMap, - data.annotation, - ); - members = Object.keys(membersMap); - expect(getCompactRow( - membersMap, - data.annotation as unknown as { [member: string]: ConfigItem }, - data.queryType as QueryType, - members, - data.query.timeDimensions as QueryTimeDimension[], - data.data[0], - )).toEqual(['95823', '646.1258666666666667']); - - data = JSON.parse( - JSON.stringify(mockData.regular_discount_by_city.data) - ); - membersMap = getMembers( - data.queryType as QueryTypeEnum, - data.query as unknown as NormalizedQuery, - data.data as { [sqlAlias: string]: DBResponseValue }[], - data.aliasToMemberNameMap, - data.annotation, - ); - members = Object.keys(membersMap); - expect(getCompactRow( - membersMap, - data.annotation as unknown as { [member: string]: ConfigItem }, - data.queryType as QueryType, - members, - data.query.timeDimensions as QueryTimeDimension[], - data.data[0], - )).toEqual(['Missouri City', '0.80000000000000000000']); - - // compare date range - data = JSON.parse( - JSON.stringify(mockData.compare_date_range_count_by_order_date.data[0]) - ); - membersMap = getMembers( - data.queryType as QueryTypeEnum, - data.query as unknown as NormalizedQuery, - data.data as { [sqlAlias: string]: DBResponseValue }[], - data.aliasToMemberNameMap, - data.annotation, - ); - members = Object.keys(membersMap); - expect(getCompactRow( - membersMap, - data.annotation as unknown as { [member: string]: ConfigItem }, - data.queryType as QueryType, - members, - data.query.timeDimensions as QueryTimeDimension[], - data.data[0], - )).toEqual([ - '2020-01-01T00:00:00.000', - '2020-01-01T00:00:00.000', - '10', - '2020-01-01T00:00:00.000 - 2020-01-31T23:59:59.999' - ]); - - data = JSON.parse( - JSON.stringify(mockData.compare_date_range_count_by_order_date.data[0]) - ); - membersMap = getMembers( - data.queryType as QueryTypeEnum, - data.query as unknown as NormalizedQuery, - data.data as { [sqlAlias: string]: DBResponseValue }[], - data.aliasToMemberNameMap, - data.annotation, - ); - members = Object.keys(membersMap); - expect(getCompactRow( - membersMap, - data.annotation as unknown as { [member: string]: ConfigItem }, - data.queryType as QueryType, - members, - data.query.timeDimensions as QueryTimeDimension[], - data.data[1], - )).toEqual([ - '2020-01-02T00:00:00.000', - '2020-01-02T00:00:00.000', - '8', - '2020-01-01T00:00:00.000 - 2020-01-31T23:59:59.999' - ]); - - // blending - data = JSON.parse( - JSON.stringify(mockData.blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode.data[0]) - ); - membersMap = getMembers( - data.queryType as QueryTypeEnum, - data.query as unknown as NormalizedQuery, - data.data as { [sqlAlias: string]: DBResponseValue }[], - data.aliasToMemberNameMap, - data.annotation, - ); - members = Object.keys(membersMap); - expect(getCompactRow( - membersMap, - data.annotation as unknown as { [member: string]: ConfigItem }, - data.queryType as QueryType, - members, - data.query.timeDimensions as QueryTimeDimension[], - data.data[0], - )).toEqual([ - '2020-01-01T00:00:00.000', - '2020-01-01T00:00:00.000', - '0.15638297872340425532', - '2020-01-01T00:00:00.000', - ]); - }); - - test('getVanilaRow helper', () => { - const data = JSON.parse( - JSON.stringify(mockData.regular_discount_by_city.data) - ); - delete data.aliasToMemberNameMap.e_commerce_records_us2021__avg_discount; - expect(() => getVanilaRow( - data.aliasToMemberNameMap, - data.annotation as unknown as { [member: string]: ConfigItem }, - data.queryType as QueryType, - data.query as unknown as NormalizedQuery, - data.data[0], - )).toThrow( - 'You requested hidden member: \'e_commerce_records_us2021__avg_discount\'. ' + - 'Please make it visible using `shown: true`. Please note ' + - 'primaryKey fields are `shown: false` by default: ' + - 'https://cube.dev/docs/schema/reference/joins#setting-a-primary-key.' - ); - }); -}); - -describe('transformData default mode', () => { - test('regular discount by city', () => { - let data; - - data = JSON.parse( - JSON.stringify(mockData.regular_discount_by_city.data) - ); - delete data.aliasToMemberNameMap.e_commerce_records_us2021__avg_discount; - expect(() => transformData( - data.aliasToMemberNameMap, - data.annotation as unknown as { [member: string]: ConfigItem }, - data.data, - data.query as unknown as NormalizedQuery, - data.queryType as QueryType, - )).toThrow(); - - data = JSON.parse( - JSON.stringify(mockData.regular_discount_by_city.data) - ); - expect( - transformData( - data.aliasToMemberNameMap, - data.annotation as unknown as { [member: string]: ConfigItem }, - data.data, - data.query as unknown as NormalizedQuery, - data.queryType as QueryType, - ) - ).toEqual(data.result_default); - }); - - test('regular profit by postal code', () => { - const data = JSON.parse( - JSON.stringify(mockData.regular_profit_by_postal_code.data) - ); - expect( - transformData( - data.aliasToMemberNameMap, - data.annotation as unknown as { [member: string]: ConfigItem }, - data.data, - data.query as unknown as NormalizedQuery, - data.queryType as QueryType, - ) - ).toEqual(data.result_default); - }); - - test('compare date range count by order date', () => { - const data = JSON.parse( - JSON.stringify(mockData.compare_date_range_count_by_order_date.data) - ); - - expect( - transformData( - data[0].aliasToMemberNameMap, - data[0].annotation as unknown as { [member: string]: ConfigItem }, - data[0].data, - data[0].query as unknown as NormalizedQuery, - data[0].queryType as QueryType, - ) - ).toEqual(data[0].result_default); - - expect( - transformData( - data[1].aliasToMemberNameMap, - data[1].annotation as unknown as { [member: string]: ConfigItem }, - data[1].data, - data[1].query as unknown as NormalizedQuery, - data[1].queryType as QueryType, - ) - ).toEqual(data[1].result_default); - }); - - test('blending query avg discount by date range for the first and standard ship mode', () => { - const data = JSON.parse( - JSON.stringify( - mockData - .blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode - .data - ) - ); - - expect( - transformData( - data[0].aliasToMemberNameMap, - data[0].annotation as unknown as { [member: string]: ConfigItem }, - data[0].data, - data[0].query as unknown as NormalizedQuery, - data[0].queryType as QueryType, - ) - ).toEqual(data[0].result_default); - - expect( - transformData( - data[1].aliasToMemberNameMap, - data[1].annotation as unknown as { [member: string]: ConfigItem }, - data[1].data, - data[1].query as unknown as NormalizedQuery, - data[1].queryType as QueryType, - ) - ).toEqual(data[1].result_default); - }); -}); - -describe('transformData compact mode', () => { - test('regular discount by city', () => { - let data; - - data = JSON.parse( - JSON.stringify(mockData.regular_discount_by_city.data) - ); - delete data.aliasToMemberNameMap.e_commerce_records_us2021__avg_discount; - expect(() => transformData( - data.aliasToMemberNameMap, - data.annotation as unknown as { [member: string]: ConfigItem }, - data.data, - data.query as unknown as NormalizedQuery, - data.queryType as QueryType, - ResultTypeEnum.COMPACT, - )).toThrow(); - - data = JSON.parse( - JSON.stringify(mockData.regular_discount_by_city.data) - ); - expect( - transformData( - data.aliasToMemberNameMap, - data.annotation as unknown as { [member: string]: ConfigItem }, - data.data, - data.query as unknown as NormalizedQuery, - data.queryType as QueryType, - ResultTypeEnum.COMPACT, - ) - ).toEqual(data.result_compact); - }); - - test('regular profit by postal code', () => { - const data = JSON.parse( - JSON.stringify(mockData.regular_profit_by_postal_code.data) - ); - expect( - transformData( - data.aliasToMemberNameMap, - data.annotation as unknown as { [member: string]: ConfigItem }, - data.data, - data.query as unknown as NormalizedQuery, - data.queryType as QueryType, - ResultTypeEnum.COMPACT, - ) - ).toEqual(data.result_compact); - }); - - test('compare date range count by order date', () => { - const data = JSON.parse( - JSON.stringify(mockData.compare_date_range_count_by_order_date.data) - ); - - expect( - transformData( - data[0].aliasToMemberNameMap, - data[0].annotation as unknown as { [member: string]: ConfigItem }, - data[0].data, - data[0].query as unknown as NormalizedQuery, - data[0].queryType as QueryType, - ResultTypeEnum.COMPACT, - ) - ).toEqual(data[0].result_compact); - - expect( - transformData( - data[1].aliasToMemberNameMap, - data[1].annotation as unknown as { [member: string]: ConfigItem }, - data[1].data, - data[1].query as unknown as NormalizedQuery, - data[1].queryType as QueryType, - ResultTypeEnum.COMPACT, - ) - ).toEqual(data[1].result_compact); - }); - - test('blending query avg discount by date range for the first and standard ship mode', () => { - const data = JSON.parse( - JSON.stringify( - mockData - .blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode - .data - ) - ); - - expect( - transformData( - data[0].aliasToMemberNameMap, - data[0].annotation as unknown as { [member: string]: ConfigItem }, - data[0].data, - data[0].query as unknown as NormalizedQuery, - data[0].queryType as QueryType, - ResultTypeEnum.COMPACT, - ) - ).toEqual(data[0].result_compact); - - expect( - transformData( - data[1].aliasToMemberNameMap, - data[1].annotation as unknown as { [member: string]: ConfigItem }, - data[1].data, - data[1].query as unknown as NormalizedQuery, - data[1].queryType as QueryType, - ResultTypeEnum.COMPACT, - ) - ).toEqual(data[1].result_compact); - }); -}); diff --git a/packages/cubejs-api-gateway/test/helpers/transformValue.test.ts b/packages/cubejs-api-gateway/test/helpers/transformValue.test.ts deleted file mode 100644 index 54101c9fcb131..0000000000000 --- a/packages/cubejs-api-gateway/test/helpers/transformValue.test.ts +++ /dev/null @@ -1,33 +0,0 @@ -/** - * @license Apache-2.0 - * @copyright Cube Dev, Inc. - * @fileoverview transformValue function unit tests. - */ - -/* globals describe,test,expect */ -/* eslint-disable import/no-duplicates */ -/* eslint-disable @typescript-eslint/no-duplicate-imports */ - -import moment from 'moment'; -import transformValueDef from '../../src/helpers/transformValue'; -import { transformValue } from '../../src/helpers/transformValue'; - -describe('transformValue helper', () => { - test('export looks as expected', () => { - expect(transformValueDef).toBeDefined(); - expect(transformValue).toBeDefined(); - expect(transformValue).toEqual(transformValueDef); - }); - test('object with the time value', () => { - const date = Date(); - expect(transformValue(date, 'time')).toBe( - moment.utc(date).format(moment.HTML5_FMT.DATETIME_LOCAL_MS) - ); - }); - test('object with the Date value', () => { - const date = new Date(); - expect(transformValue(date, 'date')).toBe( - moment(date).format(moment.HTML5_FMT.DATETIME_LOCAL_MS) - ); - }); -}); diff --git a/packages/cubejs-backend-native/Cargo.lock b/packages/cubejs-backend-native/Cargo.lock index 1ec0fc189e2b5..65113ac0e7061 100644 --- a/packages/cubejs-backend-native/Cargo.lock +++ b/packages/cubejs-backend-native/Cargo.lock @@ -128,7 +128,7 @@ dependencies = [ "chrono", "comfy-table 5.0.1", "csv", - "flatbuffers", + "flatbuffers 2.1.2", "half", "hex", "indexmap 1.9.3", @@ -176,7 +176,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] @@ -187,7 +187,7 @@ checksum = "531b97fb4cd3dfdce92c35dedbfdc1f0b9d8091c8ca943d6dae340ef5012d514" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] @@ -395,7 +395,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", "syn_derive", ] @@ -508,6 +508,7 @@ dependencies = [ "iana-time-zone", "js-sys", "num-traits", + "serde", "wasm-bindgen", "windows-targets 0.48.5", ] @@ -732,12 +733,14 @@ dependencies = [ name = "cubejs-native" version = "0.28.0" dependencies = [ + "anyhow", "async-channel", "async-trait", "axum", "bytes", "convert_case", "cubenativeutils", + "cubeorchestrator", "cubesql", "cubesqlplanner", "findshlibs", @@ -779,6 +782,26 @@ dependencies = [ "uuid 0.8.2", ] +[[package]] +name = "cubeorchestrator" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "cubeshared", + "itertools 0.13.0", + "neon", + "serde", + "serde_json", +] + +[[package]] +name = "cubeshared" +version = "0.1.0" +dependencies = [ + "flatbuffers 23.5.26", +] + [[package]] name = "cubesql" version = "0.28.0" @@ -805,7 +828,7 @@ dependencies = [ "futures-util", "hashbrown 0.14.3", "indexmap 1.9.3", - "itertools", + "itertools 0.10.5", "log", "lru", "minijinja", @@ -842,7 +865,7 @@ dependencies = [ "cubeclient", "cubenativeutils", "datafusion", - "itertools", + "itertools 0.10.5", "lazy_static", "minijinja", "nativebridge", @@ -874,7 +897,7 @@ dependencies = [ "datafusion-physical-expr", "futures", "hashbrown 0.12.3", - "itertools", + "itertools 0.10.5", "lazy_static", "log", "num_cpus", @@ -1089,6 +1112,16 @@ dependencies = [ "thiserror", ] +[[package]] +name = "flatbuffers" +version = "23.5.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640" +dependencies = [ + "bitflags 1.3.2", + "rustc_version", +] + [[package]] name = "flate2" version = "1.0.28" @@ -1200,7 +1233,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] @@ -1586,6 +1619,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.10" @@ -1903,7 +1945,7 @@ dependencies = [ "Inflector", "async-trait", "byteorder", - "itertools", + "itertools 0.10.5", "proc-macro2", "quote", "syn 1.0.109", @@ -1931,7 +1973,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6813fde79b646e47e7ad75f480aa80ef76a5d9599e2717407961531169ee38b" dependencies = [ "quote", - "syn 2.0.76", + "syn 2.0.90", "syn-mid", ] @@ -2171,7 +2213,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] @@ -2254,7 +2296,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] @@ -2388,9 +2430,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] @@ -2486,7 +2528,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] @@ -2498,7 +2540,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] @@ -2775,6 +2817,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "0.38.28" @@ -2888,29 +2939,29 @@ checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" [[package]] name = "serde" -version = "1.0.209" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.209" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] name = "serde_json" -version = "1.0.127" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad" +checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" dependencies = [ "indexmap 2.4.0", "itoa", @@ -3100,7 +3151,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] @@ -3139,9 +3190,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.76" +version = "2.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578e081a14e0cefc3279b0472138c513f37b41a08d5a3cca9b6e4e8ceb6cd525" +checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" dependencies = [ "proc-macro2", "quote", @@ -3156,7 +3207,7 @@ checksum = "b5dc35bb08dd1ca3dfb09dce91fd2d13294d6711c88897d9a9d60acf39bce049" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] @@ -3168,7 +3219,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] @@ -3247,7 +3298,7 @@ checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] @@ -3325,7 +3376,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] @@ -3468,7 +3519,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] @@ -3727,7 +3778,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", "wasm-bindgen-shared", ] @@ -3761,7 +3812,7 @@ checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4018,7 +4069,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.90", ] [[package]] diff --git a/packages/cubejs-backend-native/Cargo.toml b/packages/cubejs-backend-native/Cargo.toml index 3c3666761e1e9..37ddb4094c1c1 100644 --- a/packages/cubejs-backend-native/Cargo.toml +++ b/packages/cubejs-backend-native/Cargo.toml @@ -16,13 +16,15 @@ opt-level = 1 crate-type = ["cdylib", "lib"] [dependencies] -cubesqlplanner = { path = "../../rust/cubesqlplanner/cubesqlplanner/" } +cubesqlplanner = { path = "../../rust/cubesqlplanner/cubesqlplanner" } +cubeorchestrator = { path = "../../rust/cubeorchestrator" } cubenativeutils = { path = "../../rust/cubenativeutils" } +cubesql = { path = "../../rust/cubesql/cubesql" } +anyhow = "1.0" async-channel = { version = "2" } async-trait = "0.1.36" convert_case = "0.6.0" pin-project = "1.1.5" -cubesql = { path = "../../rust/cubesql/cubesql" } findshlibs = "0.10.2" futures = "0.3.30" http-body-util = "0.1" diff --git a/packages/cubejs-backend-native/js/ResultWrapper.ts b/packages/cubejs-backend-native/js/ResultWrapper.ts new file mode 100644 index 0000000000000..9068a83ed3a8c --- /dev/null +++ b/packages/cubejs-backend-native/js/ResultWrapper.ts @@ -0,0 +1,237 @@ +import { + getCubestoreResult, + getFinalQueryResult, + getFinalQueryResultMulti, + ResultRow +} from './index'; + +export interface DataResult { + isWrapper: boolean; + getFinalResult(): Promise; + getRawData(): any[]; + getTransformData(): any[]; + getRootResultObject(): any[]; + // eslint-disable-next-line no-use-before-define + getResults(): ResultWrapper[]; +} + +class BaseWrapper { + public readonly isWrapper: boolean = true; +} + +export class ResultWrapper extends BaseWrapper implements DataResult { + private readonly proxy: any; + + private cache: any; + + public cached: Boolean = false; + + private readonly isNative: Boolean = false; + + private readonly nativeReference: any; + + private readonly jsResult: any = null; + + private transformData: any; + + private rootResultObject: any = {}; + + public constructor(input: any) { + super(); + + if (input.isWrapper) { + return input; + } + + if (Array.isArray(input)) { + this.jsResult = input; + } else { + this.isNative = true; + this.nativeReference = input; + } + + this.proxy = new Proxy(this, { + get: (target, prop: string | symbol) => { + // To support iterative access + if (prop === Symbol.iterator) { + const array = this.getArray(); + const l = array.length; + + return function* yieldArrayItem() { + for (let i = 0; i < l; i++) { + yield array[i]; + } + }; + } + + // intercept indexes + if (typeof prop === 'string' && !Number.isNaN(Number(prop))) { + const array = this.getArray(); + return array[Number(prop)]; + } + + // intercept isNative + if (prop === 'isNative') { + return this.isNative; + } + + // intercept array props and methods + if (typeof prop === 'string' && prop in Array.prototype) { + const arrayMethod = (Array.prototype as any)[prop]; + if (typeof arrayMethod === 'function') { + return (...args: any[]) => this.invokeArrayMethod(prop, ...args); + } + + return (this.getArray() as any)[prop]; + } + + // intercept JSON.stringify or toJSON() + if (prop === 'toJSON') { + return () => this.getArray(); + } + + return (target as any)[prop]; + }, + + // intercept array length + getOwnPropertyDescriptor: (target, prop) => { + if (prop === 'length') { + const array = this.getArray(); + return { + configurable: true, + enumerable: true, + value: array.length, + writable: false + }; + } + return Object.getOwnPropertyDescriptor(target, prop); + }, + + ownKeys: (target) => { + const array = this.getArray(); + return [...Object.keys(target), ...Object.keys(array), 'length', 'isNative']; + } + }); + Object.setPrototypeOf(this.proxy, ResultWrapper.prototype); + + return this.proxy; + } + + private getArray(): ResultRow[] { + if (!this.cache) { + if (this.isNative) { + this.cache = getCubestoreResult(this.nativeReference); + } else { + this.cache = this.jsResult; + } + this.cached = true; + } + return this.cache; + } + + private invokeArrayMethod(method: string, ...args: any[]): any { + const array = this.getArray(); + return (array as any)[method](...args); + } + + public getRawData(): any[] { + if (this.isNative) { + return [this.nativeReference]; + } + + return [this.jsResult]; + } + + public setTransformData(td: any) { + this.transformData = td; + } + + public getTransformData(): any[] { + return [this.transformData]; + } + + public setRootResultObject(obj: any) { + this.rootResultObject = obj; + } + + public getRootResultObject(): any[] { + return [this.rootResultObject]; + } + + public async getFinalResult(): Promise { + return getFinalQueryResult(this.transformData, this.getRawData()[0], this.rootResultObject); + } + + public getResults(): ResultWrapper[] { + return [this]; + } +} + +class BaseWrapperArray extends BaseWrapper { + public constructor(protected readonly results: ResultWrapper[]) { + super(); + } + + protected getInternalDataArrays(): any[] { + const [transformDataJson, rawData, resultDataJson] = this.results.reduce<[Object[], any[], Object[]]>( + ([transformList, rawList, resultList], r) => { + transformList.push(r.getTransformData()[0]); + rawList.push(r.getRawData()[0]); + resultList.push(r.getRootResultObject()[0]); + return [transformList, rawList, resultList]; + }, + [[], [], []] + ); + + return [transformDataJson, rawData, resultDataJson]; + } + + // Is invoked from the native side to get + // an array of all raw wrapped results + public getResults(): ResultWrapper[] { + return this.results; + } + + public getTransformData(): any[] { + return this.results.map(r => r.getTransformData()[0]); + } + + public getRawData(): any[] { + return this.results.map(r => r.getRawData()[0]); + } + + public getRootResultObject(): any[] { + return this.results.map(r => r.getRootResultObject()[0]); + } +} + +export class ResultMultiWrapper extends BaseWrapperArray implements DataResult { + public constructor(results: ResultWrapper[], private rootResultObject: any) { + super(results); + } + + public async getFinalResult(): Promise { + const [transformDataJson, rawDataRef, cleanResultList] = this.getInternalDataArrays(); + + const responseDataObj = { + queryType: this.rootResultObject.queryType, + results: cleanResultList, + slowQuery: this.rootResultObject.slowQuery, + }; + + return getFinalQueryResultMulti(transformDataJson, rawDataRef, responseDataObj); + } +} + +// This is consumed by native side via Transport Bridge +export class ResultArrayWrapper extends BaseWrapperArray implements DataResult { + public constructor(results: ResultWrapper[]) { + super(results); + } + + public async getFinalResult(): Promise { + const [transformDataJson, rawDataRef, cleanResultList] = this.getInternalDataArrays(); + + return [transformDataJson, rawDataRef, cleanResultList]; + } +} diff --git a/packages/cubejs-backend-native/js/index.ts b/packages/cubejs-backend-native/js/index.ts index 779195ccfa8f5..f6c54477f463f 100644 --- a/packages/cubejs-backend-native/js/index.ts +++ b/packages/cubejs-backend-native/js/index.ts @@ -3,6 +3,9 @@ import fs from 'fs'; import path from 'path'; import { Writable } from 'stream'; import type { Request as ExpressRequest } from 'express'; +import { ResultWrapper } from './ResultWrapper'; + +export * from './ResultWrapper'; export interface BaseMeta { // postgres or mysql @@ -98,14 +101,28 @@ export type SQLInterfaceOptions = { gatewayPort?: number, }; +export type DBResponsePrimitive = + null | + boolean | + number | + string; + +let loadedNative: any = null; + export function loadNative() { + if (loadedNative) { + return loadedNative; + } + // Development version if (fs.existsSync(path.join(__dirname, '/../../index.node'))) { - return require(path.join(__dirname, '/../../index.node')); + loadedNative = require(path.join(__dirname, '/../../index.node')); + return loadedNative; } if (fs.existsSync(path.join(__dirname, '/../../native/index.node'))) { - return require(path.join(__dirname, '/../../native/index.node')); + loadedNative = require(path.join(__dirname, '/../../native/index.node')); + return loadedNative; } throw new Error( @@ -113,10 +130,6 @@ export function loadNative() { ); } -export function isSupported(): boolean { - return fs.existsSync(path.join(__dirname, '/../../index.node')) || fs.existsSync(path.join(__dirname, '/../../native/index.node')); -} - function wrapNativeFunctionWithChannelCallback( fn: (extra: any) => unknown | Promise, ) { @@ -253,8 +266,9 @@ function wrapNativeFunctionWithStream( }); } else if (response.error) { writerOrChannel.reject(errorString(response)); + } else if (response.isWrapper) { // Native wrapped result + writerOrChannel.resolve(response); } else { - // TODO remove JSON.stringify() writerOrChannel.resolve(JSON.stringify(response)); } } catch (e: any) { @@ -348,6 +362,49 @@ export const buildSqlAndParams = (cubeEvaluator: any): String => { return native.buildSqlAndParams(cubeEvaluator); }; +export type ResultRow = Record; + +export const parseCubestoreResultMessage = async (message: ArrayBuffer): Promise => { + const native = loadNative(); + + const msg = await native.parseCubestoreResultMessage(message); + return new ResultWrapper(msg); +}; + +export const getCubestoreResult = (ref: ResultWrapper): ResultRow[] => { + const native = loadNative(); + + return native.getCubestoreResult(ref); +}; + +/** + * Transform and prepare single query final result data that is sent to the client. + * + * @param transformDataObj Data needed to transform raw query results + * @param rows Raw data received from the source DB via driver or reference to a native CubeStore response result + * @param resultData Final query result structure without actual data + * @return {Promise} ArrayBuffer with json-serialized data which should be directly sent to the client + */ +export const getFinalQueryResult = (transformDataObj: Object, rows: any, resultData: Object): Promise => { + const native = loadNative(); + + return native.getFinalQueryResult(transformDataObj, rows, resultData); +}; + +/** + * Transform and prepare multiple query final results data into a single response structure. + * + * @param transformDataArr Array of data needed to transform raw query results + * @param rows Array of raw data received from the source DB via driver or reference to native CubeStore response results + * @param responseData Final combined query result structure without actual data + * @return {Promise} ArrayBuffer with json-serialized data which should be directly sent to the client + */ +export const getFinalQueryResultMulti = (transformDataArr: Object[], rows: any[], responseData: Object): Promise => { + const native = loadNative(); + + return native.getFinalQueryResultMulti(transformDataArr, rows, responseData); +}; + export interface PyConfiguration { repositoryFactory?: (ctx: unknown) => Promise, logger?: (msg: string, params: Record) => void, diff --git a/packages/cubejs-backend-native/src/channel.rs b/packages/cubejs-backend-native/src/channel.rs index 9394d8e9b27e3..607631c68c661 100644 --- a/packages/cubejs-backend-native/src/channel.rs +++ b/packages/cubejs-backend-native/src/channel.rs @@ -4,7 +4,9 @@ use std::collections::HashMap; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; +use crate::orchestrator::ResultWrapper; use crate::transport::MapCubeErrExt; +use crate::utils::bind_method; use async_trait::async_trait; use cubesql::transport::{SqlGenerator, SqlTemplates}; use cubesql::CubeError; @@ -13,8 +15,6 @@ use log::trace; use neon::prelude::*; use tokio::sync::oneshot; -use crate::utils::bind_method; - type JsAsyncStringChannelCallback = Box) -> Result<(), CubeError> + Send>; type JsAsyncChannelCallback = Box< @@ -194,6 +194,12 @@ where rx.await? } +#[derive(Debug)] +pub enum ValueFromJs { + String(String), + ResultWrapper(Vec), +} + #[allow(clippy::type_complexity)] pub async fn call_raw_js_with_channel_as_callback( channel: Arc, diff --git a/packages/cubejs-backend-native/src/lib.rs b/packages/cubejs-backend-native/src/lib.rs index 07a36aa69481a..68285075541b6 100644 --- a/packages/cubejs-backend-native/src/lib.rs +++ b/packages/cubejs-backend-native/src/lib.rs @@ -11,7 +11,9 @@ pub mod cross; pub mod gateway; pub mod logger; pub mod node_export; +pub mod node_obj_deserializer; pub mod node_obj_serializer; +pub mod orchestrator; #[cfg(feature = "python")] pub mod python; pub mod stream; diff --git a/packages/cubejs-backend-native/src/node_export.rs b/packages/cubejs-backend-native/src/node_export.rs index 53464958d20b2..5d4eb733206c6 100644 --- a/packages/cubejs-backend-native/src/node_export.rs +++ b/packages/cubejs-backend-native/src/node_export.rs @@ -9,11 +9,6 @@ use futures::StreamExt; use serde_json::Map; use tokio::sync::Semaphore; -use std::net::SocketAddr; -use std::rc::Rc; -use std::str::FromStr; -use std::sync::Arc; - use crate::auth::{NativeAuthContext, NodeBridgeAuthService}; use crate::channel::call_js_fn; use crate::config::{NodeConfiguration, NodeConfigurationFactoryOptions, NodeCubeServices}; @@ -31,6 +26,10 @@ use cubenativeutils::wrappers::serializer::NativeDeserialize; use cubenativeutils::wrappers::NativeContextHolder; use cubesqlplanner::cube_bridge::base_query_options::NativeBaseQueryOptions; use cubesqlplanner::planner::base_query::BaseQuery; +use std::net::SocketAddr; +use std::rc::Rc; +use std::str::FromStr; +use std::sync::Arc; use cubesql::{telemetry::ReportingLogger, CubeError}; @@ -513,8 +512,12 @@ pub fn register_module_exports( cx.export_function("isFallbackBuild", is_fallback_build)?; cx.export_function("__js_to_clrepr_to_js", debug_js_to_clrepr_to_js)?; + //============ sql planner exports =================== cx.export_function("buildSqlAndParams", build_sql_and_params)?; + //========= sql orchestrator exports ================= + crate::orchestrator::register_module(&mut cx)?; + crate::template::template_register_module(&mut cx)?; #[cfg(feature = "python")] diff --git a/packages/cubejs-backend-native/src/node_obj_deserializer.rs b/packages/cubejs-backend-native/src/node_obj_deserializer.rs new file mode 100644 index 0000000000000..d9ed815682792 --- /dev/null +++ b/packages/cubejs-backend-native/src/node_obj_deserializer.rs @@ -0,0 +1,436 @@ +use neon::prelude::*; +use neon::result::Throw; +use serde::de::{ + self, Deserializer, EnumAccess, IntoDeserializer, MapAccess, SeqAccess, StdError, + VariantAccess, Visitor, +}; +use serde::forward_to_deserialize_any; +use std::fmt; + +#[derive(Debug)] +pub struct JsDeserializationError(String); + +impl fmt::Display for JsDeserializationError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "JS Deserialization Error: {}", self.0) + } +} + +impl From for JsDeserializationError { + fn from(throw: Throw) -> Self { + JsDeserializationError(throw.to_string()) + } +} + +impl StdError for JsDeserializationError {} + +impl de::Error for JsDeserializationError { + fn custom(msg: T) -> Self { + JsDeserializationError(msg.to_string()) + } +} + +pub struct JsValueDeserializer<'a, 'b> { + pub cx: &'a mut FunctionContext<'b>, + pub value: Handle<'a, JsValue>, +} + +impl<'a, 'b> JsValueDeserializer<'a, 'b> { + pub fn new(cx: &'a mut FunctionContext<'b>, value: Handle<'a, JsValue>) -> Self { + Self { cx, value } + } +} + +impl<'de, 'a, 'b> Deserializer<'de> for JsValueDeserializer<'a, 'b> { + type Error = JsDeserializationError; + + fn deserialize_any(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + if self.value.is_a::(self.cx) { + let value = self + .value + .downcast::(self.cx) + .or_throw(self.cx)? + .value(self.cx); + visitor.visit_string(value) + } else if self.value.is_a::(self.cx) { + let value = self + .value + .downcast::(self.cx) + .or_throw(self.cx)? + .value(self.cx); + + // floats + if value.fract() != 0.0 { + return visitor.visit_f64(value); + } + + // unsigned + if value >= 0.0 { + if value <= u8::MAX as f64 { + return visitor.visit_u8(value as u8); + } + + if value <= u16::MAX as f64 { + return visitor.visit_u16(value as u16); + } + + if value <= u32::MAX as f64 { + return visitor.visit_u32(value as u32); + } + + if value <= u64::MAX as f64 { + return visitor.visit_u64(value as u64); + } + } + + if value >= i8::MIN as f64 && value <= i8::MAX as f64 { + return visitor.visit_i8(value as i8); + } + + if value >= i16::MIN as f64 && value <= i16::MAX as f64 { + return visitor.visit_i16(value as i16); + } + + if value >= i32::MIN as f64 && value <= i32::MAX as f64 { + return visitor.visit_i32(value as i32); + } + + if value >= i64::MIN as f64 && value <= i64::MAX as f64 { + return visitor.visit_i64(value as i64); + } + + Err(JsDeserializationError( + "Unsupported number type for deserialization".to_string(), + )) + } else if self.value.is_a::(self.cx) { + let value = self + .value + .downcast::(self.cx) + .or_throw(self.cx)? + .value(self.cx); + visitor.visit_bool(value) + } else if self.value.is_a::(self.cx) { + let js_array = self + .value + .downcast::(self.cx) + .or_throw(self.cx)?; + let deserializer = JsArrayDeserializer::new(self.cx, js_array); + visitor.visit_seq(deserializer) + } else if self.value.is_a::(self.cx) { + let js_object = self + .value + .downcast::(self.cx) + .or_throw(self.cx)?; + let deserializer = JsObjectDeserializer::new(self.cx, js_object); + visitor.visit_map(deserializer) + } else if self.value.is_a::(self.cx) + || self.value.is_a::(self.cx) + { + visitor.visit_none() + } else if self.value.is_a::(self.cx) { + // We can do nothing with the JS functions in native + visitor.visit_none() + } else { + Err(JsDeserializationError( + "Unsupported type for deserialization".to_string(), + )) + } + } + + fn deserialize_option(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + if self.value.is_a::(self.cx) || self.value.is_a::(self.cx) { + visitor.visit_none() + } else { + visitor.visit_some(self) + } + } + + fn deserialize_unit(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_unit() + } + + fn deserialize_unit_struct( + self, + _name: &'static str, + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + self.deserialize_unit(visitor) + } + + fn deserialize_newtype_struct( + self, + _name: &'static str, + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + visitor.visit_newtype_struct(self) + } + + fn deserialize_seq(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + if self.value.is_a::(self.cx) { + let js_array = self + .value + .downcast::(self.cx) + .or_throw(self.cx)?; + let deserializer = JsArrayDeserializer::new(self.cx, js_array); + visitor.visit_seq(deserializer) + } else { + Err(JsDeserializationError("expected an array".to_string())) + } + } + + fn deserialize_tuple(self, _len: usize, visitor: V) -> Result + where + V: Visitor<'de>, + { + self.deserialize_seq(visitor) + } + + fn deserialize_tuple_struct( + self, + _name: &'static str, + _len: usize, + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + self.deserialize_seq(visitor) + } + + fn deserialize_map(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + if self.value.is_a::(self.cx) { + let js_object = self + .value + .downcast::(self.cx) + .or_throw(self.cx)?; + let deserializer = JsObjectDeserializer::new(self.cx, js_object); + visitor.visit_map(deserializer) + } else { + Err(JsDeserializationError("expected an object".to_string())) + } + } + + fn deserialize_struct( + self, + _name: &'static str, + _fields: &'static [&'static str], + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + self.deserialize_map(visitor) + } + + fn deserialize_enum( + self, + _name: &'static str, + _variants: &'static [&'static str], + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + let deserializer = JsEnumDeserializer::new(self.cx, self.value); + visitor.visit_enum(deserializer) + } + + forward_to_deserialize_any! { + bool i8 i16 i32 i64 u8 u16 u32 u64 f32 f64 char str string + bytes byte_buf identifier ignored_any + } +} + +struct JsObjectDeserializer<'a, 'b> { + cx: &'a mut FunctionContext<'b>, + js_object: Handle<'a, JsObject>, + keys: Vec, + index: usize, +} + +impl<'a, 'b> JsObjectDeserializer<'a, 'b> { + fn new(cx: &'a mut FunctionContext<'b>, js_object: Handle<'a, JsObject>) -> Self { + let keys = js_object + .get_own_property_names(cx) + .expect("Failed to get object keys") + .to_vec(cx) + .expect("Failed to convert keys to Vec") + .iter() + .filter_map(|k| { + k.downcast_or_throw::(cx) + .ok() + .map(|js_string| js_string.value(cx)) + }) + .collect::>(); + Self { + cx, + js_object, + keys, + index: 0, + } + } +} + +// `MapAccess` is provided to the `Visitor` to give it the ability to iterate +// through entries of the map. +impl<'de, 'a, 'b> MapAccess<'de> for JsObjectDeserializer<'a, 'b> { + type Error = JsDeserializationError; + + fn next_key_seed(&mut self, seed: K) -> Result, Self::Error> + where + K: de::DeserializeSeed<'de>, + { + if self.index >= self.keys.len() { + return Ok(None); + } + let key = &self.keys[self.index]; + self.index += 1; + seed.deserialize(key.as_str().into_deserializer()).map(Some) + } + + fn next_value_seed(&mut self, seed: V) -> Result + where + V: de::DeserializeSeed<'de>, + { + let key = &self.keys[self.index - 1]; + let value = self + .js_object + .get(self.cx, key.as_str()) + .expect("Failed to get value by key"); + seed.deserialize(JsValueDeserializer::new(self.cx, value)) + } +} + +struct JsArrayDeserializer<'a, 'b> { + cx: &'a mut FunctionContext<'b>, + js_array: Handle<'a, JsArray>, + index: usize, + length: usize, +} + +impl<'a, 'b> JsArrayDeserializer<'a, 'b> { + fn new(cx: &'a mut FunctionContext<'b>, js_array: Handle<'a, JsArray>) -> Self { + let length = js_array.len(cx) as usize; + Self { + cx, + js_array, + index: 0, + length, + } + } +} + +// `SeqAccess` is provided to the `Visitor` to give it the ability to iterate +// through elements of the sequence. +impl<'de, 'a, 'b> SeqAccess<'de> for JsArrayDeserializer<'a, 'b> { + type Error = JsDeserializationError; + + fn next_element_seed(&mut self, seed: T) -> Result, Self::Error> + where + T: de::DeserializeSeed<'de>, + { + if self.index >= self.length { + return Ok(None); + } + let value = self + .js_array + .get(self.cx, self.index as u32) + .map_err(JsDeserializationError::from)?; + self.index += 1; + + seed.deserialize(JsValueDeserializer::new(self.cx, value)) + .map(Some) + } + + fn size_hint(&self) -> Option { + Some(self.length) + } +} + +pub struct JsEnumDeserializer<'a, 'b> { + cx: &'a mut FunctionContext<'b>, + value: Handle<'a, JsValue>, +} + +impl<'a, 'b> JsEnumDeserializer<'a, 'b> { + fn new(cx: &'a mut FunctionContext<'b>, value: Handle<'a, JsValue>) -> Self { + Self { cx, value } + } +} + +// `EnumAccess` is provided to the `Visitor` to give it the ability to determine +// which variant of the enum is supposed to be deserialized. +impl<'de, 'a, 'b> EnumAccess<'de> for JsEnumDeserializer<'a, 'b> { + type Error = JsDeserializationError; + type Variant = Self; + + fn variant_seed(self, seed: T) -> Result<(T::Value, Self::Variant), Self::Error> + where + T: de::DeserializeSeed<'de>, + { + let variant = seed.deserialize(JsValueDeserializer::new(self.cx, self.value))?; + Ok((variant, self)) + } +} + +impl<'de, 'a, 'b> VariantAccess<'de> for JsEnumDeserializer<'a, 'b> { + type Error = JsDeserializationError; + + // If the `Visitor` expected this variant to be a unit variant, the input + // should have been the plain string case handled in `deserialize_enum`. + fn unit_variant(self) -> Result<(), Self::Error> { + Ok(()) + } + + // Newtype variants are represented in JSON as `{ NAME: VALUE }` so + // deserialize the value here. + fn newtype_variant_seed(self, seed: T) -> Result + where + T: de::DeserializeSeed<'de>, + { + seed.deserialize(JsValueDeserializer::new(self.cx, self.value)) + } + + // Tuple variants are represented in JSON as `{ NAME: [DATA...] }` so + // deserialize the sequence of data here. + fn tuple_variant(self, _len: usize, visitor: V) -> Result + where + V: Visitor<'de>, + { + Deserializer::deserialize_seq(JsValueDeserializer::new(self.cx, self.value), visitor) + } + + // Struct variants are represented in JSON as `{ NAME: { K: V, ... } }` so + // deserialize the inner map here. + fn struct_variant( + self, + _fields: &'static [&'static str], + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + Deserializer::deserialize_map(JsValueDeserializer::new(self.cx, self.value), visitor) + } +} diff --git a/packages/cubejs-backend-native/src/orchestrator.rs b/packages/cubejs-backend-native/src/orchestrator.rs new file mode 100644 index 0000000000000..227e8b9970d76 --- /dev/null +++ b/packages/cubejs-backend-native/src/orchestrator.rs @@ -0,0 +1,346 @@ +use crate::node_obj_deserializer::JsValueDeserializer; +use crate::transport::MapCubeErrExt; +use cubeorchestrator::query_message_parser::QueryResult; +use cubeorchestrator::query_result_transform::{ + DBResponsePrimitive, RequestResultData, RequestResultDataMulti, TransformedData, +}; +use cubeorchestrator::transport::{JsRawData, TransformDataRequest}; +use cubesql::compile::engine::df::scan::{FieldValue, ValueObject}; +use cubesql::CubeError; +use neon::context::{Context, FunctionContext, ModuleContext}; +use neon::handle::Handle; +use neon::object::Object; +use neon::prelude::{ + JsArray, JsArrayBuffer, JsBox, JsBuffer, JsFunction, JsObject, JsPromise, JsResult, JsValue, + NeonResult, +}; +use neon::types::buffer::TypedArray; +use serde::Deserialize; +use std::borrow::Cow; +use std::sync::Arc; + +pub fn register_module(cx: &mut ModuleContext) -> NeonResult<()> { + cx.export_function( + "parseCubestoreResultMessage", + parse_cubestore_result_message, + )?; + cx.export_function("getCubestoreResult", get_cubestore_result)?; + cx.export_function("getFinalQueryResult", final_query_result)?; + cx.export_function("getFinalQueryResultMulti", final_query_result_multi)?; + + Ok(()) +} + +#[derive(Debug, Clone)] +pub struct ResultWrapper { + transform_data: TransformDataRequest, + data: Arc, + transformed_data: Option, +} + +impl ResultWrapper { + pub fn from_js_result_wrapper( + cx: &mut FunctionContext<'_>, + js_result_wrapper_val: Handle, + ) -> Result { + let js_result_wrapper = js_result_wrapper_val + .downcast::(cx) + .map_cube_err("Can't downcast JS ResultWrapper to object")?; + + let get_transform_data_js_method: Handle = js_result_wrapper + .get(cx, "getTransformData") + .map_cube_err("Can't get getTransformData() method from JS ResultWrapper object")?; + + let transform_data_js_arr = get_transform_data_js_method + .call(cx, js_result_wrapper.upcast::(), []) + .map_cube_err("Error calling getTransformData() method of ResultWrapper object")? + .downcast::(cx) + .map_cube_err("Can't downcast JS transformData to array")? + .to_vec(cx) + .map_cube_err("Can't convert JS transformData to array")?; + + let transform_data_js = transform_data_js_arr.first().unwrap(); + + let deserializer = JsValueDeserializer::new(cx, *transform_data_js); + let transform_request: TransformDataRequest = match Deserialize::deserialize(deserializer) { + Ok(data) => data, + Err(_) => { + return Err(CubeError::internal( + "Can't deserialize transformData from JS ResultWrapper object".to_string(), + )) + } + }; + + let get_raw_data_js_method: Handle = js_result_wrapper + .get(cx, "getRawData") + .map_cube_err("Can't get getRawData() method from JS ResultWrapper object")?; + + let raw_data_js_arr = get_raw_data_js_method + .call(cx, js_result_wrapper.upcast::(), []) + .map_cube_err("Error calling getRawData() method of ResultWrapper object")? + .downcast::(cx) + .map_cube_err("Can't downcast JS rawData to array")? + .to_vec(cx) + .map_cube_err("Can't convert JS rawData to array")?; + + let raw_data_js = raw_data_js_arr.first().unwrap(); + + let query_result = + if let Ok(js_box) = raw_data_js.downcast::>, _>(cx) { + Arc::clone(&js_box) + } else if let Ok(js_array) = raw_data_js.downcast::(cx) { + let deserializer = JsValueDeserializer::new(cx, js_array.upcast()); + let js_raw_data: JsRawData = match Deserialize::deserialize(deserializer) { + Ok(data) => data, + Err(_) => { + return Err(CubeError::internal( + "Can't deserialize results raw data from JS ResultWrapper object" + .to_string(), + )); + } + }; + + QueryResult::from_js_raw_data(js_raw_data) + .map(Arc::new) + .map_cube_err("Can't build results data from JS rawData")? + } else { + return Err(CubeError::internal( + "Can't deserialize results raw data from JS ResultWrapper object".to_string(), + )); + }; + + Ok(Self { + transform_data: transform_request, + data: query_result, + transformed_data: None, + }) + } + + pub fn transform_result(&mut self) -> Result<(), CubeError> { + self.transformed_data = Some( + TransformedData::transform(&self.transform_data, &self.data) + .map_cube_err("Can't prepare transformed data")?, + ); + + Ok(()) + } +} + +impl ValueObject for ResultWrapper { + fn len(&mut self) -> Result { + if self.transformed_data.is_none() { + self.transform_result()?; + } + + let data = self.transformed_data.as_ref().unwrap(); + + match data { + TransformedData::Compact { + members: _members, + dataset, + } => Ok(dataset.len()), + TransformedData::Vanilla(dataset) => Ok(dataset.len()), + } + } + + fn get(&mut self, index: usize, field_name: &str) -> Result { + if self.transformed_data.is_none() { + self.transform_result()?; + } + + let data = self.transformed_data.as_ref().unwrap(); + + let value = match data { + TransformedData::Compact { members, dataset } => { + let Some(row) = dataset.get(index) else { + return Err(CubeError::user(format!( + "Unexpected response from Cube, can't get {} row", + index + ))); + }; + + let Some(member_index) = members.iter().position(|m| m == field_name) else { + return Err(CubeError::user(format!( + "Field name '{}' not found in members", + field_name + ))); + }; + + row.get(member_index).unwrap_or(&DBResponsePrimitive::Null) + } + TransformedData::Vanilla(dataset) => { + let Some(row) = dataset.get(index) else { + return Err(CubeError::user(format!( + "Unexpected response from Cube, can't get {} row", + index + ))); + }; + + row.get(field_name).unwrap_or(&DBResponsePrimitive::Null) + } + }; + + Ok(match value { + DBResponsePrimitive::String(s) => FieldValue::String(Cow::Borrowed(s)), + DBResponsePrimitive::Number(n) => FieldValue::Number(*n), + DBResponsePrimitive::Boolean(b) => FieldValue::Bool(*b), + DBResponsePrimitive::Null => FieldValue::Null, + }) + } +} + +fn json_to_array_buffer<'a, C>( + mut cx: C, + json_data: Result, +) -> JsResult<'a, JsArrayBuffer> +where + C: Context<'a>, +{ + match json_data { + Ok(json_data) => { + let json_bytes = json_data.as_bytes(); + let mut js_buffer = cx.array_buffer(json_bytes.len())?; + { + let buffer = js_buffer.as_mut_slice(&mut cx); + buffer.copy_from_slice(json_bytes); + } + Ok(js_buffer) + } + Err(err) => cx.throw_error(err.to_string()), + } +} + +fn extract_query_result( + cx: &mut FunctionContext<'_>, + data_arg: Handle, +) -> Result, anyhow::Error> { + if let Ok(js_box) = data_arg.downcast::>, _>(cx) { + Ok(Arc::clone(&js_box)) + } else if let Ok(js_array) = data_arg.downcast::(cx) { + let deserializer = JsValueDeserializer::new(cx, js_array.upcast()); + let js_raw_data: JsRawData = Deserialize::deserialize(deserializer)?; + + QueryResult::from_js_raw_data(js_raw_data) + .map(Arc::new) + .map_err(anyhow::Error::from) + } else { + Err(anyhow::anyhow!( + "Second argument must be an Array of JsBox> or JsArray" + )) + } +} + +pub fn parse_cubestore_result_message(mut cx: FunctionContext) -> JsResult { + let msg = cx.argument::(0)?; + let msg_data = msg.as_slice(&cx).to_vec(); + + let promise = cx + .task(move || QueryResult::from_cubestore_fb(&msg_data)) + .promise(move |mut cx, res| match res { + Ok(result) => Ok(cx.boxed(Arc::new(result))), + Err(err) => cx.throw_error(err.to_string()), + }); + + Ok(promise) +} + +pub fn get_cubestore_result(mut cx: FunctionContext) -> JsResult { + let result = cx.argument::>>(0)?; + + let js_array = cx.execute_scoped(|mut cx| { + let js_array = JsArray::new(&mut cx, result.rows.len()); + + for (i, row) in result.rows.iter().enumerate() { + let js_row = cx.execute_scoped(|mut cx| { + let js_row = JsObject::new(&mut cx); + for (key, value) in result.columns.iter().zip(row.iter()) { + let js_key = cx.string(key); + let js_value = cx.string(value.to_string()); + js_row.set(&mut cx, js_key, js_value)?; + } + Ok(js_row) + })?; + + js_array.set(&mut cx, i as u32, js_row)?; + } + + Ok(js_array) + })?; + + Ok(js_array.upcast()) +} + +pub fn final_query_result(mut cx: FunctionContext) -> JsResult { + let transform_data_js_object = cx.argument::(0)?; + let deserializer = JsValueDeserializer::new(&mut cx, transform_data_js_object); + let transform_request_data: TransformDataRequest = match Deserialize::deserialize(deserializer) + { + Ok(data) => data, + Err(err) => return cx.throw_error(err.to_string()), + }; + + let data_arg = cx.argument::(1)?; + let cube_store_result: Arc = match extract_query_result(&mut cx, data_arg) { + Ok(query_result) => query_result, + Err(err) => return cx.throw_error(err.to_string()), + }; + + let result_data_js_object = cx.argument::(2)?; + let deserializer = JsValueDeserializer::new(&mut cx, result_data_js_object); + let mut result_data: RequestResultData = match Deserialize::deserialize(deserializer) { + Ok(data) => data, + Err(err) => return cx.throw_error(err.to_string()), + }; + + let promise = cx + .task(move || { + result_data.prepare_results(&transform_request_data, &cube_store_result)?; + + match serde_json::to_string(&result_data) { + Ok(json) => Ok(json), + Err(err) => Err(anyhow::Error::from(err)), + } + }) + .promise(move |cx, json_data| json_to_array_buffer(cx, json_data)); + + Ok(promise) +} + +pub fn final_query_result_multi(mut cx: FunctionContext) -> JsResult { + let transform_data_array = cx.argument::(0)?; + let deserializer = JsValueDeserializer::new(&mut cx, transform_data_array); + let transform_requests: Vec = match Deserialize::deserialize(deserializer) + { + Ok(data) => data, + Err(err) => return cx.throw_error(err.to_string()), + }; + + let data_array = cx.argument::(1)?; + let mut cube_store_results: Vec> = vec![]; + for data_arg in data_array.to_vec(&mut cx)? { + match extract_query_result(&mut cx, data_arg) { + Ok(query_result) => cube_store_results.push(query_result), + Err(err) => return cx.throw_error(err.to_string()), + }; + } + + let result_data_js_object = cx.argument::(2)?; + let deserializer = JsValueDeserializer::new(&mut cx, result_data_js_object); + let mut result_data: RequestResultDataMulti = match Deserialize::deserialize(deserializer) { + Ok(data) => data, + Err(err) => return cx.throw_error(err.to_string()), + }; + + let promise = cx + .task(move || { + result_data.prepare_results(&transform_requests, &cube_store_results)?; + + match serde_json::to_string(&result_data) { + Ok(json) => Ok(json), + Err(err) => Err(anyhow::Error::from(err)), + } + }) + .promise(move |cx, json_data| json_to_array_buffer(cx, json_data)); + + Ok(promise) +} diff --git a/packages/cubejs-backend-native/src/transport.rs b/packages/cubejs-backend-native/src/transport.rs index 30c5fbc1b84e5..a54aa3f46bda3 100644 --- a/packages/cubejs-backend-native/src/transport.rs +++ b/packages/cubejs-backend-native/src/transport.rs @@ -3,8 +3,18 @@ use neon::prelude::*; use std::collections::HashMap; use std::fmt::Display; +use crate::auth::NativeAuthContext; +use crate::channel::{call_raw_js_with_channel_as_callback, NodeSqlGenerator, ValueFromJs}; +use crate::node_obj_serializer::NodeObjSerializer; +use crate::orchestrator::ResultWrapper; +use crate::{ + auth::TransportRequest, channel::call_js_with_channel_as_callback, + stream::call_js_with_stream_as_callback, +}; use async_trait::async_trait; -use cubesql::compile::engine::df::scan::{MemberField, SchemaRef}; +use cubesql::compile::engine::df::scan::{ + convert_transport_response, transform_response, MemberField, RecordBatch, SchemaRef, +}; use cubesql::compile::engine::df::wrapper::SqlQuery; use cubesql::transport::{ SpanId, SqlGenerator, SqlResponse, TransportLoadRequestQuery, TransportLoadResponse, @@ -20,14 +30,6 @@ use serde::Serialize; use std::sync::Arc; use uuid::Uuid; -use crate::auth::NativeAuthContext; -use crate::channel::{call_raw_js_with_channel_as_callback, NodeSqlGenerator}; -use crate::node_obj_serializer::NodeObjSerializer; -use crate::{ - auth::TransportRequest, channel::call_js_with_channel_as_callback, - stream::call_js_with_stream_as_callback, -}; - #[derive(Debug)] pub struct NodeBridgeTransport { channel: Arc, @@ -334,7 +336,9 @@ impl TransportService for NodeBridgeTransport { sql_query: Option, ctx: AuthContextRef, meta: LoadRequestMeta, - ) -> Result { + schema: SchemaRef, + member_fields: Vec, + ) -> Result, CubeError> { trace!("[transport] Request ->"); let native_auth = ctx @@ -369,54 +373,115 @@ impl TransportService for NodeBridgeTransport { streaming: false, })?; - let result = call_js_with_channel_as_callback( + let result = call_raw_js_with_channel_as_callback( self.channel.clone(), self.on_sql_api_load.clone(), - Some(extra), + extra, + Box::new(|cx, v| Ok(cx.string(v).as_value(cx))), + Box::new(move |cx, v| { + if let Ok(js_result_wrapped) = v.downcast::(cx) { + let get_results_js_method: Handle = + js_result_wrapped.get(cx, "getResults").map_cube_err( + "Can't get getResults() method from JS ResultWrapper object", + )?; + + let results = get_results_js_method + .call(cx, js_result_wrapped.upcast::(), []) + .map_cube_err( + "Error calling getResults() method of ResultWrapper object", + )?; + + let js_res_wrapped_vec = results + .downcast::(cx) + .map_cube_err("Can't downcast JS result to array")? + .to_vec(cx) + .map_cube_err("Can't convert JS result to array")?; + + let native_wrapped_results = js_res_wrapped_vec + .iter() + .map(|r| ResultWrapper::from_js_result_wrapper(cx, *r)) + .collect::, _>>() + .map_cube_err( + "Can't construct result wrapper from JS ResultWrapper object", + )?; + + Ok(ValueFromJs::ResultWrapper(native_wrapped_results)) + } else if let Ok(str) = v.downcast::(cx) { + Ok(ValueFromJs::String(str.value(cx))) + } else { + Err(CubeError::internal( + "Can't downcast callback argument to string or resultWrapper object" + .to_string(), + )) + } + }), ) .await; + if let Err(e) = &result { if e.message.to_lowercase().contains("continue wait") { continue; } } - let response: serde_json::Value = result?; - - #[cfg(debug_assertions)] - trace!("[transport] Request <- {:?}", response); - #[cfg(not(debug_assertions))] - trace!("[transport] Request <- "); - - if let Some(error_value) = response.get("error") { - match error_value { - serde_json::Value::String(error) => { - if error.to_lowercase() == *"continue wait" { - debug!( + match result? { + ValueFromJs::String(result) => { + let response: serde_json::Value = match serde_json::from_str(&result) { + Ok(json) => json, + Err(err) => return Err(CubeError::internal(err.to_string())), + }; + + #[cfg(debug_assertions)] + trace!("[transport] Request <- {:?}", response); + #[cfg(not(debug_assertions))] + trace!("[transport] Request <- "); + + if let Some(error_value) = response.get("error") { + match error_value { + serde_json::Value::String(error) => { + if error.to_lowercase() == *"continue wait" { + debug!( "[transport] load - retrying request (continue wait) requestId: {}", request_id ); - continue; - } else { - return Err(CubeError::user(error.clone())); - } - } - other => { - error!( + continue; + } else { + return Err(CubeError::user(error.clone())); + } + } + other => { + error!( "[transport] load - strange response, success which contains error: {:?}", other ); - return Err(CubeError::internal( - "Error response with broken data inside".to_string(), - )); - } - } - }; + return Err(CubeError::internal( + "Error response with broken data inside".to_string(), + )); + } + } + }; - break serde_json::from_value::(response) - .map_err(|err| CubeError::user(err.to_string())); + let response = match serde_json::from_value::(response) { + Ok(v) => v, + Err(err) => { + return Err(CubeError::user(err.to_string())); + } + }; + + break convert_transport_response(response, schema.clone(), member_fields) + .map_err(|err| CubeError::user(err.to_string())); + } + ValueFromJs::ResultWrapper(result_wrappers) => { + break result_wrappers + .into_iter() + .map(|mut wrapper| { + transform_response(&mut wrapper, schema.clone(), &member_fields) + }) + .collect::, _>>(); + } + } } } diff --git a/packages/cubejs-backend-native/test/sql.test.ts b/packages/cubejs-backend-native/test/sql.test.ts index 1d55187483dd7..e417b577bc842 100644 --- a/packages/cubejs-backend-native/test/sql.test.ts +++ b/packages/cubejs-backend-native/test/sql.test.ts @@ -6,7 +6,7 @@ import * as native from '../js'; import metaFixture from './meta'; import { FakeRowStream } from './response-fake'; -const logger = jest.fn(({ event }) => { +const _logger = jest.fn(({ event }) => { if ( !event.error.includes( 'load - strange response, success which contains error' diff --git a/packages/cubejs-backend-shared/package.json b/packages/cubejs-backend-shared/package.json index 691caf3fdee2e..5e31b8527a507 100644 --- a/packages/cubejs-backend-shared/package.json +++ b/packages/cubejs-backend-shared/package.json @@ -27,7 +27,7 @@ "@types/cli-progress": "^3.9.1", "@types/decompress": "^4.2.3", "@types/jest": "^27", - "@types/node": "^12", + "@types/node": "^18", "@types/node-fetch": "^2.5.8", "@types/shelljs": "^0.8.5", "@types/throttle-debounce": "^2.1.0", diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index 4d4a2e97462a3..33d1854ff7c73 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -193,6 +193,9 @@ const variables: Record any> = { .default('1') .asInt(), nativeSqlPlanner: () => get('CUBEJS_TESSERACT_SQL_PLANNER').asBool(), + nativeOrchestrator: () => get('CUBEJS_TESSERACT_ORCHESTRATOR') + .default('false') + .asBoolStrict(), /** **************************************************************** * Common db options * diff --git a/packages/cubejs-cubestore-driver/package.json b/packages/cubejs-cubestore-driver/package.json index 125e69e55acbf..8a4a7bc8db9c2 100644 --- a/packages/cubejs-cubestore-driver/package.json +++ b/packages/cubejs-cubestore-driver/package.json @@ -29,6 +29,7 @@ "@cubejs-backend/base-driver": "1.1.16", "@cubejs-backend/cubestore": "1.1.12", "@cubejs-backend/shared": "1.1.12", + "@cubejs-backend/native": "1.1.16", "csv-write-stream": "^2.0.0", "flatbuffers": "23.3.3", "fs-extra": "^9.1.0", diff --git a/packages/cubejs-cubestore-driver/src/WebSocketConnection.ts b/packages/cubejs-cubestore-driver/src/WebSocketConnection.ts index 92da79e88ca2d..94170ed8ffd36 100644 --- a/packages/cubejs-cubestore-driver/src/WebSocketConnection.ts +++ b/packages/cubejs-cubestore-driver/src/WebSocketConnection.ts @@ -1,8 +1,9 @@ import WebSocket from 'ws'; import * as flatbuffers from 'flatbuffers'; +import { v4 as uuidv4 } from 'uuid'; import { InlineTable } from '@cubejs-backend/base-driver'; import { getEnv } from '@cubejs-backend/shared'; -import { v4 as uuidv4 } from 'uuid'; +import { parseCubestoreResultMessage } from '@cubejs-backend/native'; import { HttpCommand, HttpError, @@ -108,7 +109,7 @@ export class WebSocketConnection { this.webSocket = undefined; } }); - webSocket.on('message', (msg) => { + webSocket.on('message', async (msg) => { const buf = new flatbuffers.ByteBuffer(msg); const httpMessage = HttpMessage.getRootAsHttpMessage(buf); const resolvers = webSocket.sentMessages[httpMessage.messageId()]; @@ -116,44 +117,59 @@ export class WebSocketConnection { if (!resolvers) { throw new Error(`Cube Store missed message id: ${httpMessage.messageId()}`); // logging } - const commandType = httpMessage.commandType(); - if (commandType === HttpCommand.HttpError) { - resolvers.reject(new Error(`${httpMessage.command(new HttpError())?.error()}`)); - } else if (commandType === HttpCommand.HttpResultSet) { - const resultSet = httpMessage.command(new HttpResultSet()); - if (!resultSet) { - resolvers.reject(new Error('Empty resultSet')); - return; + + if (getEnv('nativeOrchestrator') && msg.length > 1000) { + try { + const nativeResMsg = await parseCubestoreResultMessage(msg); + resolvers.resolve(nativeResMsg); + } catch (e) { + resolvers.reject(e); } - const columnsLen = resultSet.columnsLength(); - const columns: Array = []; - for (let i = 0; i < columnsLen; i++) { - const columnName = resultSet.columns(i); - if (!columnName) { - resolvers.reject(new Error('Column name is not defined')); + } else { + const commandType = httpMessage.commandType(); + + if (commandType === HttpCommand.HttpError) { + resolvers.reject(new Error(`${httpMessage.command(new HttpError())?.error()}`)); + } else if (commandType === HttpCommand.HttpResultSet) { + const resultSet = httpMessage.command(new HttpResultSet()); + + if (!resultSet) { + resolvers.reject(new Error('Empty resultSet')); return; } - columns.push(columnName); - } - const rowLen = resultSet.rowsLength(); - const result: any[] = []; - for (let i = 0; i < rowLen; i++) { - const row = resultSet.rows(i); - if (!row) { - resolvers.reject(new Error('Null row')); - return; + + const columnsLen = resultSet.columnsLength(); + const columns: Array = []; + for (let i = 0; i < columnsLen; i++) { + const columnName = resultSet.columns(i); + if (!columnName) { + resolvers.reject(new Error('Column name is not defined')); + return; + } + columns.push(columnName); } - const valueLen = row.valuesLength(); - const rowObj = {}; - for (let j = 0; j < valueLen; j++) { - const value = row.values(j); - rowObj[columns[j]] = value?.stringValue(); + + const rowLen = resultSet.rowsLength(); + const result: any[] = []; + for (let i = 0; i < rowLen; i++) { + const row = resultSet.rows(i); + if (!row) { + resolvers.reject(new Error('Null row')); + return; + } + const valueLen = row.valuesLength(); + const rowObj = {}; + for (let j = 0; j < valueLen; j++) { + const value = row.values(j); + rowObj[columns[j]] = value?.stringValue(); + } + result.push(rowObj); } - result.push(rowObj); + + resolvers.resolve(result); + } else { + resolvers.reject(new Error('Unsupported command')); } - resolvers.resolve(result); - } else { - resolvers.reject(new Error('Unsupported command')); } }); }); diff --git a/packages/cubejs-mssql-driver/package.json b/packages/cubejs-mssql-driver/package.json index 4bd76fec90535..9ae7ce48d650c 100644 --- a/packages/cubejs-mssql-driver/package.json +++ b/packages/cubejs-mssql-driver/package.json @@ -18,7 +18,7 @@ }, "devDependencies": { "@types/mssql": "^9.1.5", - "@types/node": "^16" + "@types/node": "^18" }, "jest": { "testEnvironment": "node" diff --git a/packages/cubejs-server/src/websocket-server.ts b/packages/cubejs-server/src/websocket-server.ts index 6e4b101ccd23d..cca6c9b44c4c7 100644 --- a/packages/cubejs-server/src/websocket-server.ts +++ b/packages/cubejs-server/src/websocket-server.ts @@ -31,12 +31,26 @@ export class WebSocketServer { const connectionIdToSocket: Record = {}; - this.subscriptionServer = this.serverCore.initSubscriptionServer((connectionId: string, message: any) => { + this.subscriptionServer = this.serverCore.initSubscriptionServer(async (connectionId: string, message: any) => { if (!connectionIdToSocket[connectionId]) { throw new Error(`Socket for ${connectionId} is not found found`); } - connectionIdToSocket[connectionId].send(JSON.stringify(message)); + let messageStr: string; + + if (message.message && message.message.isWrapper) { + // In case we have a wrapped query result, we don't want to parse/stringify + // it again - it's too expensive, instead we serialize the rest of the message and then + // inject query result json into message. + const resMsg = new TextDecoder().decode(await message.message.getFinalResult()); + message.message = '~XXXXX~'; + messageStr = JSON.stringify(message); + messageStr = messageStr.replace('"~XXXXX~"', resMsg); + } else { + messageStr = JSON.stringify(message); + } + + connectionIdToSocket[connectionId].send(messageStr); }); this.wsServer.on('connection', (ws) => { diff --git a/rust/cubenativeutils/Cargo.lock b/rust/cubenativeutils/Cargo.lock index 0959a0f2888e6..59b5872a1fd3c 100644 --- a/rust/cubenativeutils/Cargo.lock +++ b/rust/cubenativeutils/Cargo.lock @@ -165,7 +165,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", ] [[package]] @@ -176,7 +176,7 @@ checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", ] [[package]] @@ -326,7 +326,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", "syn_derive", ] @@ -1048,7 +1048,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", ] [[package]] @@ -1726,7 +1726,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6813fde79b646e47e7ad75f480aa80ef76a5d9599e2717407961531169ee38b" dependencies = [ "quote", - "syn 2.0.68", + "syn 2.0.95", "syn-mid", ] @@ -1963,7 +1963,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", ] [[package]] @@ -2046,7 +2046,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", ] [[package]] @@ -2165,9 +2165,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] @@ -2580,32 +2580,33 @@ checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" [[package]] name = "serde" -version = "1.0.209" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.209" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", ] [[package]] name = "serde_json" -version = "1.0.120" +version = "1.0.135" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" +checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" dependencies = [ "indexmap 2.2.6", "itoa", + "memchr", "ryu", "serde", ] @@ -2770,7 +2771,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.68", + "syn 2.0.95", ] [[package]] @@ -2809,9 +2810,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.68" +version = "2.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" +checksum = "46f71c0377baf4ef1cc3e3402ded576dccc315800fbc62dfc7fe04b009773b4a" dependencies = [ "proc-macro2", "quote", @@ -2826,7 +2827,7 @@ checksum = "b5dc35bb08dd1ca3dfb09dce91fd2d13294d6711c88897d9a9d60acf39bce049" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", ] [[package]] @@ -2838,7 +2839,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", ] [[package]] @@ -2904,7 +2905,7 @@ checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", ] [[package]] @@ -2972,7 +2973,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", ] [[package]] @@ -3074,7 +3075,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", ] [[package]] @@ -3308,7 +3309,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", "wasm-bindgen-shared", ] @@ -3342,7 +3343,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3596,7 +3597,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.95", ] [[package]] diff --git a/rust/cubenativeutils/rustfmt.toml b/rust/cubenativeutils/rustfmt.toml new file mode 100644 index 0000000000000..d9ba5fdb90ba3 --- /dev/null +++ b/rust/cubenativeutils/rustfmt.toml @@ -0,0 +1 @@ +imports_granularity = "Crate" \ No newline at end of file diff --git a/rust/cubenativeutils/src/wrappers/context.rs b/rust/cubenativeutils/src/wrappers/context.rs index ba880a17eff6a..98bb3ba6f7eb9 100644 --- a/rust/cubenativeutils/src/wrappers/context.rs +++ b/rust/cubenativeutils/src/wrappers/context.rs @@ -1,5 +1,4 @@ -use super::inner_types::InnerTypes; -use super::object_handle::NativeObjectHandle; +use super::{inner_types::InnerTypes, object_handle::NativeObjectHandle}; pub trait NativeContext: Clone { fn boolean(&self, v: bool) -> IT::Boolean; diff --git a/rust/cubenativeutils/src/wrappers/inner_types.rs b/rust/cubenativeutils/src/wrappers/inner_types.rs index 24586a56234a5..66bfbf136375c 100644 --- a/rust/cubenativeutils/src/wrappers/inner_types.rs +++ b/rust/cubenativeutils/src/wrappers/inner_types.rs @@ -1,7 +1,9 @@ -use super::context::NativeContext; -use super::object::{ - NativeArray, NativeBoolean, NativeFunction, NativeNumber, NativeObject, NativeString, - NativeStruct, +use super::{ + context::NativeContext, + object::{ + NativeArray, NativeBoolean, NativeFunction, NativeNumber, NativeObject, NativeString, + NativeStruct, + }, }; pub trait InnerTypes: Clone + 'static { type Object: NativeObject; diff --git a/rust/cubenativeutils/src/wrappers/neon/context.rs b/rust/cubenativeutils/src/wrappers/neon/context.rs index e641ab5c20296..96950e5e3aedb 100644 --- a/rust/cubenativeutils/src/wrappers/neon/context.rs +++ b/rust/cubenativeutils/src/wrappers/neon/context.rs @@ -1,18 +1,21 @@ //use super::object::NeonObject; -use super::inner_types::NeonInnerTypes; -use super::object::base_types::*; -use super::object::neon_array::NeonArray; -use super::object::neon_function::NeonFunction; -use super::object::neon_struct::NeonStruct; -use super::object::NeonObject; -use crate::wrappers::context::NativeContext; -use crate::wrappers::object::NativeObject; -use crate::wrappers::object_handle::NativeObjectHandle; +use super::{ + inner_types::NeonInnerTypes, + object::{ + base_types::*, neon_array::NeonArray, neon_function::NeonFunction, neon_struct::NeonStruct, + NeonObject, + }, +}; +use crate::wrappers::{ + context::NativeContext, object::NativeObject, object_handle::NativeObjectHandle, +}; use cubesql::CubeError; use neon::prelude::*; -use std::cell::{RefCell, RefMut}; -use std::marker::PhantomData; -use std::rc::{Rc, Weak}; +use std::{ + cell::{RefCell, RefMut}, + marker::PhantomData, + rc::{Rc, Weak}, +}; pub struct ContextWrapper<'cx, C: Context<'cx>> { cx: C, lifetime: PhantomData<&'cx ()>, diff --git a/rust/cubenativeutils/src/wrappers/neon/inner_types.rs b/rust/cubenativeutils/src/wrappers/neon/inner_types.rs index ebdfdf6f6279f..8bf32d0be843e 100644 --- a/rust/cubenativeutils/src/wrappers/neon/inner_types.rs +++ b/rust/cubenativeutils/src/wrappers/neon/inner_types.rs @@ -1,9 +1,10 @@ -use super::context::ContextHolder; -use super::object::base_types::*; -use super::object::neon_array::NeonArray; -use super::object::neon_function::NeonFunction; -use super::object::neon_struct::NeonStruct; -use super::object::NeonObject; +use super::{ + context::ContextHolder, + object::{ + base_types::*, neon_array::NeonArray, neon_function::NeonFunction, neon_struct::NeonStruct, + NeonObject, + }, +}; use crate::wrappers::inner_types::InnerTypes; use neon::prelude::*; use std::marker::PhantomData; diff --git a/rust/cubenativeutils/src/wrappers/neon/object/mod.rs b/rust/cubenativeutils/src/wrappers/neon/object/mod.rs index cb263bfeb6ea3..40763a228b97d 100644 --- a/rust/cubenativeutils/src/wrappers/neon/object/mod.rs +++ b/rust/cubenativeutils/src/wrappers/neon/object/mod.rs @@ -3,13 +3,14 @@ pub mod neon_array; pub mod neon_function; pub mod neon_struct; -use self::base_types::{NeonBoolean, NeonNumber, NeonString}; -use self::neon_array::NeonArray; -use self::neon_function::NeonFunction; -use self::neon_struct::NeonStruct; +use self::{ + base_types::{NeonBoolean, NeonNumber, NeonString}, + neon_array::NeonArray, + neon_function::NeonFunction, + neon_struct::NeonStruct, +}; use super::inner_types::NeonInnerTypes; -use crate::wrappers::neon::context::ContextHolder; -use crate::wrappers::object::NativeObject; +use crate::wrappers::{neon::context::ContextHolder, object::NativeObject}; use cubesql::CubeError; use neon::prelude::*; diff --git a/rust/cubenativeutils/src/wrappers/neon/object/neon_array.rs b/rust/cubenativeutils/src/wrappers/neon/object/neon_array.rs index 85bf544555b1a..dc06714227c61 100644 --- a/rust/cubenativeutils/src/wrappers/neon/object/neon_array.rs +++ b/rust/cubenativeutils/src/wrappers/neon/object/neon_array.rs @@ -1,7 +1,9 @@ use super::NeonObject; -use crate::wrappers::neon::inner_types::NeonInnerTypes; -use crate::wrappers::object::{NativeArray, NativeObject, NativeType}; -use crate::wrappers::object_handle::NativeObjectHandle; +use crate::wrappers::{ + neon::inner_types::NeonInnerTypes, + object::{NativeArray, NativeObject, NativeType}, + object_handle::NativeObjectHandle, +}; use cubesql::CubeError; use neon::prelude::*; diff --git a/rust/cubenativeutils/src/wrappers/neon/object/neon_function.rs b/rust/cubenativeutils/src/wrappers/neon/object/neon_function.rs index 78f663ff365a9..c8fc68d372d50 100644 --- a/rust/cubenativeutils/src/wrappers/neon/object/neon_function.rs +++ b/rust/cubenativeutils/src/wrappers/neon/object/neon_function.rs @@ -1,7 +1,9 @@ use super::NeonObject; -use crate::wrappers::neon::inner_types::NeonInnerTypes; -use crate::wrappers::object::{NativeFunction, NativeType}; -use crate::wrappers::object_handle::NativeObjectHandle; +use crate::wrappers::{ + neon::inner_types::NeonInnerTypes, + object::{NativeFunction, NativeType}, + object_handle::NativeObjectHandle, +}; use cubesql::CubeError; use lazy_static::lazy_static; use neon::prelude::*; diff --git a/rust/cubenativeutils/src/wrappers/neon/object/neon_struct.rs b/rust/cubenativeutils/src/wrappers/neon/object/neon_struct.rs index 7815a19f306f0..693c19aeb1da3 100644 --- a/rust/cubenativeutils/src/wrappers/neon/object/neon_struct.rs +++ b/rust/cubenativeutils/src/wrappers/neon/object/neon_struct.rs @@ -1,7 +1,9 @@ use super::NeonObject; -use crate::wrappers::neon::inner_types::NeonInnerTypes; -use crate::wrappers::object::{NativeStruct, NativeType}; -use crate::wrappers::object_handle::NativeObjectHandle; +use crate::wrappers::{ + neon::inner_types::NeonInnerTypes, + object::{NativeStruct, NativeType}, + object_handle::NativeObjectHandle, +}; use cubesql::CubeError; use neon::prelude::*; diff --git a/rust/cubenativeutils/src/wrappers/object.rs b/rust/cubenativeutils/src/wrappers/object.rs index f901ededed639..bee3275ea3aac 100644 --- a/rust/cubenativeutils/src/wrappers/object.rs +++ b/rust/cubenativeutils/src/wrappers/object.rs @@ -1,5 +1,4 @@ -use super::inner_types::InnerTypes; -use super::object_handle::NativeObjectHandle; +use super::{inner_types::InnerTypes, object_handle::NativeObjectHandle}; use cubesql::CubeError; pub trait NativeObject: Clone { diff --git a/rust/cubenativeutils/src/wrappers/object_handle.rs b/rust/cubenativeutils/src/wrappers/object_handle.rs index 7dffd313b8b96..501c16e356be8 100644 --- a/rust/cubenativeutils/src/wrappers/object_handle.rs +++ b/rust/cubenativeutils/src/wrappers/object_handle.rs @@ -1,5 +1,4 @@ -use super::inner_types::InnerTypes; -use super::object::NativeObject; +use super::{inner_types::InnerTypes, object::NativeObject}; use cubesql::CubeError; #[derive(Clone)] diff --git a/rust/cubenativeutils/src/wrappers/serializer/deserialize.rs b/rust/cubenativeutils/src/wrappers/serializer/deserialize.rs index 7576e9578e733..3f38afab3a592 100644 --- a/rust/cubenativeutils/src/wrappers/serializer/deserialize.rs +++ b/rust/cubenativeutils/src/wrappers/serializer/deserialize.rs @@ -1,7 +1,8 @@ use super::deserializer::NativeSerdeDeserializer; -use crate::wrappers::inner_types::InnerTypes; -use crate::wrappers::NativeObjectHandle; -use crate::CubeError; +use crate::{ + wrappers::{inner_types::InnerTypes, NativeObjectHandle}, + CubeError, +}; use serde::de::DeserializeOwned; pub trait NativeDeserialize: Sized { diff --git a/rust/cubenativeutils/src/wrappers/serializer/deserializer.rs b/rust/cubenativeutils/src/wrappers/serializer/deserializer.rs index d98984fa2412e..9f3cc78582723 100644 --- a/rust/cubenativeutils/src/wrappers/serializer/deserializer.rs +++ b/rust/cubenativeutils/src/wrappers/serializer/deserializer.rs @@ -1,13 +1,14 @@ use super::error::NativeObjSerializerError; -use crate::wrappers::inner_types::InnerTypes; -use crate::wrappers::object::{ - NativeArray, NativeBoolean, NativeNumber, NativeString, NativeStruct, +use crate::wrappers::{ + inner_types::InnerTypes, + object::{NativeArray, NativeBoolean, NativeNumber, NativeString, NativeStruct}, + object_handle::NativeObjectHandle, +}; +use serde::{ + self, + de::{DeserializeOwned, DeserializeSeed, MapAccess, SeqAccess, Visitor}, + forward_to_deserialize_any, Deserializer, }; -use crate::wrappers::object_handle::NativeObjectHandle; -use serde; -use serde::de::{DeserializeOwned, DeserializeSeed, MapAccess, SeqAccess, Visitor}; -use serde::forward_to_deserialize_any; -use serde::Deserializer; pub struct NativeSerdeDeserializer { input: NativeObjectHandle, diff --git a/rust/cubenativeutils/src/wrappers/serializer/error.rs b/rust/cubenativeutils/src/wrappers/serializer/error.rs index 219b219ca7050..50e55c88da9f4 100644 --- a/rust/cubenativeutils/src/wrappers/serializer/error.rs +++ b/rust/cubenativeutils/src/wrappers/serializer/error.rs @@ -1,6 +1,5 @@ use serde::{de, ser}; -use std::fmt; -use std::fmt::Display; +use std::{fmt, fmt::Display}; #[derive(Debug)] pub enum NativeObjSerializerError { Message(String), diff --git a/rust/cubenativeutils/src/wrappers/serializer/serialize.rs b/rust/cubenativeutils/src/wrappers/serializer/serialize.rs index 1dce7a117d432..e08ef8284053f 100644 --- a/rust/cubenativeutils/src/wrappers/serializer/serialize.rs +++ b/rust/cubenativeutils/src/wrappers/serializer/serialize.rs @@ -1,7 +1,8 @@ use super::serializer::NativeSerdeSerializer; -use crate::wrappers::inner_types::InnerTypes; -use crate::wrappers::{NativeContextHolder, NativeObjectHandle}; -use crate::CubeError; +use crate::{ + wrappers::{inner_types::InnerTypes, NativeContextHolder, NativeObjectHandle}, + CubeError, +}; use serde::Serialize; pub trait NativeSerialize { diff --git a/rust/cubenativeutils/src/wrappers/serializer/serializer.rs b/rust/cubenativeutils/src/wrappers/serializer/serializer.rs index a1e04657f977c..21463875cddd2 100644 --- a/rust/cubenativeutils/src/wrappers/serializer/serializer.rs +++ b/rust/cubenativeutils/src/wrappers/serializer/serializer.rs @@ -1,7 +1,8 @@ use super::error::NativeObjSerializerError; -use crate::wrappers::inner_types::InnerTypes; -use crate::wrappers::NativeContextHolder; -use crate::wrappers::{NativeArray, NativeObjectHandle, NativeString, NativeStruct, NativeType}; +use crate::wrappers::{ + inner_types::InnerTypes, NativeArray, NativeContextHolder, NativeObjectHandle, NativeString, + NativeStruct, NativeType, +}; use serde::{ser, Serialize}; pub struct NativeSerdeSerializer { diff --git a/rust/cubeorchestrator/.gitignore b/rust/cubeorchestrator/.gitignore new file mode 100644 index 0000000000000..2a0a960cb1ec5 --- /dev/null +++ b/rust/cubeorchestrator/.gitignore @@ -0,0 +1,3 @@ +/target +/.idea +.vscode diff --git a/rust/cubeorchestrator/CHANGELOG.md b/rust/cubeorchestrator/CHANGELOG.md new file mode 100644 index 0000000000000..f05ecfa971fe9 --- /dev/null +++ b/rust/cubeorchestrator/CHANGELOG.md @@ -0,0 +1,2 @@ +# ChangeLog + diff --git a/rust/cubeorchestrator/Cargo.lock b/rust/cubeorchestrator/Cargo.lock new file mode 100644 index 0000000000000..f4a1156cfff88 --- /dev/null +++ b/rust/cubeorchestrator/Cargo.lock @@ -0,0 +1,537 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61f2b7f93d2c7d2b08263acaa4a363b3e276806c68af6134c44f523bf1aacd" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "backtrace" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a905d892734eea339e896738c14b9afce22b5318f64b951e70bf3844419b01" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "cc" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17f6e324229dc011159fcc089755d1e2e216a90d43a7dea6853ca740b84f35e7" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-targets", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cubeorchestrator" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "cubeshared", + "itertools", + "neon", + "serde", + "serde_json", +] + +[[package]] +name = "cubeshared" +version = "0.1.0" +dependencies = [ + "flatbuffers", +] + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + +[[package]] +name = "flatbuffers" +version = "23.5.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640" +dependencies = [ + "bitflags", + "rustc_version", +] + +[[package]] +name = "gimli" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0a01e0497841a3b2db4f8afa483cce65f7e96a3498bd6c541734792aeac8fe7" + +[[package]] +name = "iana-time-zone" +version = "0.1.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" + +[[package]] +name = "js-sys" +version = "0.3.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a865e038f7f6ed956f788f0d7d60c541fff74c7bd74272c5d4cf15c63743e705" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.153" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" + +[[package]] +name = "libloading" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" +dependencies = [ + "cfg-if", + "windows-targets", +] + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "memchr" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" + +[[package]] +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +dependencies = [ + "adler", + "autocfg", +] + +[[package]] +name = "neon" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d75440242411c87dc39847b0e33e961ec1f10326a9d8ecf9c1ea64a3b3c13dc" +dependencies = [ + "libloading", + "neon-macros", + "once_cell", + "semver", + "send_wrapper", + "smallvec", + "tokio", +] + +[[package]] +name = "neon-macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6813fde79b646e47e7ad75f480aa80ef76a5d9599e2717407961531169ee38b" +dependencies = [ + "quote", + "syn", + "syn-mid", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "object" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c55827317fb4c08822499848a14237d2874d6f139828893017237e7ab93eb386" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "pin-project-lite" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" + +[[package]] +name = "proc-macro2" +version = "1.0.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dead70b0b5e03e9c814bcb6b01e03e68f7c57a80aa48c72ec92152ab3e818d49" + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "semver" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" + +[[package]] +name = "send_wrapper" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" + +[[package]] +name = "serde" +version = "1.0.215" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.215" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.133" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "syn" +version = "2.0.90" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn-mid" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5dc35bb08dd1ca3dfb09dce91fd2d13294d6711c88897d9a9d60acf39bce049" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio" +version = "1.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" +dependencies = [ + "backtrace", + "pin-project-lite", +] + +[[package]] +name = "unicode-ident" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" + +[[package]] +name = "wasm-bindgen" +version = "0.2.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d15e63b4482863c109d70a7b8706c1e364eb6ea449b201a76c5b89cedcec2d5c" +dependencies = [ + "cfg-if", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d36ef12e3aaca16ddd3f67922bc63e48e953f126de60bd33ccc0101ef9998cd" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "705440e08b42d3e4b36de7d66c944be628d579796b8090bfa3471478a2260051" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98c9ae5a76e46f4deecd0f0255cc223cfa18dc9b261213b8aa0c7b36f61b3f1d" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ee99da9c5ba11bd675621338ef6fa52296b76b83305e9b6e5c77d4c286d6d49" + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" diff --git a/rust/cubeorchestrator/Cargo.toml b/rust/cubeorchestrator/Cargo.toml new file mode 100644 index 0000000000000..0605d4608ae08 --- /dev/null +++ b/rust/cubeorchestrator/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "cubeorchestrator" +version = "0.1.0" +edition = "2021" + +[dependencies] +chrono = { version = "0.4.31", features = ["serde"] } +cubeshared = { path = "../cubeshared" } +serde = { version = "1.0.215", features = ["derive"] } +serde_json = "1.0.133" +anyhow = "1.0" +itertools = "0.13.0" + +[dependencies.neon] +version = "=1" +default-features = false +features = ["napi-1", "napi-4", "napi-6", "futures"] diff --git a/rust/cubeorchestrator/rust-toolchain.toml b/rust/cubeorchestrator/rust-toolchain.toml new file mode 100644 index 0000000000000..040357e9b1d43 --- /dev/null +++ b/rust/cubeorchestrator/rust-toolchain.toml @@ -0,0 +1,5 @@ +[toolchain] +#channel = "stable" +channel = "nightly-2024-07-15" +components = ["rustfmt", "clippy"] +profile = "minimal" diff --git a/rust/cubeorchestrator/rustfmt.toml b/rust/cubeorchestrator/rustfmt.toml new file mode 100644 index 0000000000000..c3c8c37533810 --- /dev/null +++ b/rust/cubeorchestrator/rustfmt.toml @@ -0,0 +1 @@ +imports_granularity = "Crate" diff --git a/rust/cubeorchestrator/src/lib.rs b/rust/cubeorchestrator/src/lib.rs new file mode 100644 index 0000000000000..03f0453c16db7 --- /dev/null +++ b/rust/cubeorchestrator/src/lib.rs @@ -0,0 +1,3 @@ +pub mod query_message_parser; +pub mod query_result_transform; +pub mod transport; diff --git a/rust/cubeorchestrator/src/query_message_parser.rs b/rust/cubeorchestrator/src/query_message_parser.rs new file mode 100644 index 0000000000000..8d406906ceda1 --- /dev/null +++ b/rust/cubeorchestrator/src/query_message_parser.rs @@ -0,0 +1,146 @@ +use crate::{ + query_result_transform::{DBResponsePrimitive, DBResponseValue}, + transport::JsRawData, +}; +use cubeshared::codegen::{root_as_http_message, HttpCommand}; +use neon::prelude::Finalize; +use std::collections::HashMap; + +#[derive(Debug)] +pub enum ParseError { + UnsupportedCommand, + EmptyResultSet, + NullRow, + ColumnNameNotDefined, + FlatBufferError, + ErrorMessage(String), +} + +impl std::fmt::Display for ParseError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ParseError::UnsupportedCommand => write!(f, "Unsupported command"), + ParseError::EmptyResultSet => write!(f, "Empty resultSet"), + ParseError::NullRow => write!(f, "Null row"), + ParseError::ColumnNameNotDefined => write!(f, "Column name is not defined"), + ParseError::FlatBufferError => write!(f, "FlatBuffer parsing error"), + ParseError::ErrorMessage(msg) => write!(f, "Error: {}", msg), + } + } +} + +impl std::error::Error for ParseError {} + +#[derive(Debug, Clone)] +pub struct QueryResult { + pub columns: Vec, + pub rows: Vec>, + pub columns_pos: HashMap, +} + +impl Finalize for QueryResult {} + +impl QueryResult { + pub fn from_cubestore_fb(msg_data: &[u8]) -> Result { + let mut result = QueryResult { + columns: vec![], + rows: vec![], + columns_pos: HashMap::new(), + }; + + let http_message = + root_as_http_message(msg_data).map_err(|_| ParseError::FlatBufferError)?; + + match http_message.command_type() { + HttpCommand::HttpError => { + let http_error = http_message + .command_as_http_error() + .ok_or(ParseError::FlatBufferError)?; + let error_message = http_error.error().unwrap_or("Unknown error").to_string(); + Err(ParseError::ErrorMessage(error_message)) + } + HttpCommand::HttpResultSet => { + let result_set = http_message + .command_as_http_result_set() + .ok_or(ParseError::EmptyResultSet)?; + + if let Some(result_set_columns) = result_set.columns() { + if result_set_columns.iter().any(|c| c.is_empty()) { + return Err(ParseError::ColumnNameNotDefined); + } + + let (columns, columns_pos): (Vec<_>, HashMap<_, _>) = result_set_columns + .iter() + .enumerate() + .map(|(index, column_name)| { + (column_name.to_owned(), (column_name.to_owned(), index)) + }) + .unzip(); + + result.columns = columns; + result.columns_pos = columns_pos; + } + + if let Some(result_set_rows) = result_set.rows() { + result.rows = Vec::with_capacity(result_set_rows.len()); + + for row in result_set_rows.iter() { + let values = row.values().ok_or(ParseError::NullRow)?; + let row_obj: Vec<_> = values + .iter() + .map(|val| { + DBResponseValue::Primitive(DBResponsePrimitive::String( + val.string_value().unwrap_or("").to_owned(), + )) + }) + .collect(); + + result.rows.push(row_obj); + } + } + + Ok(result) + } + _ => Err(ParseError::UnsupportedCommand), + } + } + + pub fn from_js_raw_data(js_raw_data: JsRawData) -> Result { + if js_raw_data.is_empty() { + return Ok(QueryResult { + columns: vec![], + rows: vec![], + columns_pos: HashMap::new(), + }); + } + + let first_row = &js_raw_data[0]; + let columns: Vec = first_row.keys().cloned().collect(); + let columns_pos: HashMap = columns + .iter() + .enumerate() + .map(|(index, column)| (column.clone(), index)) + .collect(); + + let rows: Vec> = js_raw_data + .into_iter() + .map(|row_map| { + columns + .iter() + .map(|col| { + row_map + .get(col) + .map(|val| DBResponseValue::Primitive(val.clone())) + .unwrap_or(DBResponseValue::Primitive(DBResponsePrimitive::Null)) + }) + .collect() + }) + .collect(); + + Ok(QueryResult { + columns, + rows, + columns_pos, + }) + } +} diff --git a/rust/cubeorchestrator/src/query_result_transform.rs b/rust/cubeorchestrator/src/query_result_transform.rs new file mode 100644 index 0000000000000..22e18a889327b --- /dev/null +++ b/rust/cubeorchestrator/src/query_result_transform.rs @@ -0,0 +1,2500 @@ +use crate::{ + query_message_parser::QueryResult, + transport::{ + AnnotatedConfigItem, ConfigItem, MemberOrMemberExpression, MembersMap, NormalizedQuery, + QueryTimeDimension, QueryType, ResultType, TransformDataRequest, + }, +}; +use anyhow::{bail, Context, Result}; +use chrono::{DateTime, NaiveDateTime, TimeZone, Utc}; +use itertools::multizip; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::{ + collections::{HashMap, HashSet}, + fmt::Display, + sync::Arc, +}; + +pub const COMPARE_DATE_RANGE_FIELD: &str = "compareDateRange"; +pub const COMPARE_DATE_RANGE_SEPARATOR: &str = " - "; +pub const BLENDING_QUERY_KEY_PREFIX: &str = "time."; +pub const BLENDING_QUERY_RES_SEPARATOR: &str = "."; +pub const MEMBER_SEPARATOR: &str = "."; + +/// Transform specified `value` with specified `type` to the network protocol type. +pub fn transform_value(value: DBResponseValue, type_: &str) -> DBResponsePrimitive { + match value { + DBResponseValue::DateTime(dt) if type_ == "time" || type_.is_empty() => { + DBResponsePrimitive::String( + dt.with_timezone(&Utc) + .format("%Y-%m-%dT%H:%M:%S%.3f") + .to_string(), + ) + } + DBResponseValue::Primitive(DBResponsePrimitive::String(ref s)) if type_ == "time" => { + let formatted = DateTime::parse_from_rfc3339(s) + .map(|dt| dt.format("%Y-%m-%dT%H:%M:%S%.3f").to_string()) + .or_else(|_| { + NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S%.3f").map(|dt| { + Utc.from_utc_datetime(&dt) + .format("%Y-%m-%dT%H:%M:%S%.3f") + .to_string() + }) + }) + .or_else(|_| { + NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S").map(|dt| { + Utc.from_utc_datetime(&dt) + .format("%Y-%m-%dT%H:%M:%S%.3f") + .to_string() + }) + }) + .or_else(|_| { + NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S").map(|dt| { + Utc.from_utc_datetime(&dt) + .format("%Y-%m-%dT%H:%M:%S%.3f") + .to_string() + }) + }) + .or_else(|_| { + NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S%.3f %Z").map(|dt| { + Utc.from_utc_datetime(&dt) + .format("%Y-%m-%dT%H:%M:%S%.3f") + .to_string() + }) + }) + .or_else(|_| { + NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S%.3f %:z").map(|dt| { + Utc.from_utc_datetime(&dt) + .format("%Y-%m-%dT%H:%M:%S%.3f") + .to_string() + }) + }) + .unwrap_or_else(|_| s.clone()); + DBResponsePrimitive::String(formatted) + } + DBResponseValue::Primitive(p) => p, + DBResponseValue::Object { value } => value, + _ => DBResponsePrimitive::Null, + } +} + +/// Parse date range value from time dimension. +pub fn get_date_range_value( + time_dimensions: Option<&Vec>, +) -> Result { + let time_dimensions = match time_dimensions { + Some(time_dimensions) => time_dimensions, + None => bail!("QueryTimeDimension should be specified for the compare date range query."), + }; + + let dim = match time_dimensions.first() { + Some(dim) => dim, + None => bail!("No time dimension provided."), + }; + + let date_range: &Vec = match &dim.date_range { + Some(date_range) => date_range, + None => bail!("Inconsistent QueryTimeDimension configuration: dateRange required."), + }; + + if date_range.len() == 1 { + bail!( + "Inconsistent dateRange configuration for the compare date range query: {}", + date_range[0] + ); + } + + Ok(DBResponsePrimitive::String( + date_range.join(COMPARE_DATE_RANGE_SEPARATOR), + )) +} + +/// Parse blending query key from time dimension for query. +pub fn get_blending_query_key(time_dimensions: Option<&Vec>) -> Result { + let dim = time_dimensions + .and_then(|dims| dims.first().cloned()) + .context("QueryTimeDimension should be specified for the blending query.")?; + + let granularity = dim + .granularity.clone() + .context(format!( + "Inconsistent QueryTimeDimension configuration for the blending query, granularity required: {:?}", + dim + ))?; + + Ok(format!("{}{}", BLENDING_QUERY_KEY_PREFIX, granularity)) +} + +/// Parse blending query key from time dimension for response. +pub fn get_blending_response_key( + time_dimensions: Option<&Vec>, +) -> Result { + let dim = time_dimensions + .and_then(|dims| dims.first().cloned()) + .context("QueryTimeDimension should be specified for the blending query.")?; + + let granularity = dim + .granularity.clone() + .context(format!( + "Inconsistent QueryTimeDimension configuration for the blending query, granularity required: {:?}", + dim + ))?; + + let dimension = dim.dimension.clone(); + + Ok(format!( + "{}{}{}", + dimension, BLENDING_QUERY_RES_SEPARATOR, granularity + )) +} + +/// Parse member names from request/response. +pub fn get_members( + query_type: &QueryType, + query: &NormalizedQuery, + db_data: &QueryResult, + alias_to_member_name_map: &HashMap, + annotation: &HashMap, +) -> Result<(MembersMap, Vec)> { + let mut members_map: MembersMap = HashMap::new(); + // Hashmaps don't guarantee the order of the elements while iterating + // this fires in get_compact_row because members map doesn't hold the members for + // date range queries, which are added later and thus columns in final recordset are not + // in sync with the order of members in members list. + let mut members_arr: Vec = vec![]; + + if db_data.columns.is_empty() { + return Ok((members_map, members_arr)); + } + + for column in db_data.columns.iter() { + let member_name = alias_to_member_name_map + .get(column) + .context(format!("Member name not found for alias: '{}'", column))?; + + if !annotation.contains_key(member_name) { + bail!( + concat!( + "You requested hidden member: '{}'. Please make it visible using `shown: true`. ", + "Please note primaryKey fields are `shown: false` by default: ", + "https://cube.dev/docs/schema/reference/joins#setting-a-primary-key." + ), + column + ); + } + + members_map.insert(member_name.clone(), column.clone()); + members_arr.push(member_name.clone()); + + let path = member_name.split(MEMBER_SEPARATOR).collect::>(); + let calc_member = format!("{}{}{}", path[0], MEMBER_SEPARATOR, path[1]); + + if path.len() == 3 + && query.dimensions.as_ref().map_or(true, |dims| { + !dims + .iter() + .any(|dim| *dim == MemberOrMemberExpression::Member(calc_member.clone())) + }) + { + members_map.insert(calc_member.clone(), column.clone()); + members_arr.push(calc_member); + } + } + + match query_type { + QueryType::CompareDateRangeQuery => { + members_map.insert( + COMPARE_DATE_RANGE_FIELD.to_string(), + QueryType::CompareDateRangeQuery.to_string(), + ); + members_arr.push(COMPARE_DATE_RANGE_FIELD.to_string()); + } + QueryType::BlendingQuery => { + let blending_key = get_blending_query_key(query.time_dimensions.as_ref()) + .context("Failed to generate blending query key")?; + if let Some(dim) = query + .time_dimensions + .as_ref() + .and_then(|dims| dims.first().cloned()) + { + let val = members_map.get(&dim.dimension).unwrap(); + members_map.insert(blending_key.clone(), val.clone()); + members_arr.push(blending_key); + } + } + _ => {} + } + + Ok((members_map, members_arr)) +} + +/// Convert DB response object to the compact output format. +pub fn get_compact_row( + members_to_alias_map: &HashMap, + annotation: &HashMap, + query_type: &QueryType, + members: &[String], + time_dimensions: Option<&Vec>, + db_row: &[DBResponseValue], + columns_pos: &HashMap, +) -> Result> { + let mut row: Vec = Vec::with_capacity(members.len()); + + for m in members { + if let Some(annotation_item) = annotation.get(m) { + if let Some(alias) = members_to_alias_map.get(m) { + if let Some(key) = columns_pos.get(alias) { + if let Some(value) = db_row.get(*key) { + let mtype = annotation_item.member_type.as_deref().unwrap_or(""); + row.push(transform_value(value.clone(), mtype)); + } + } + } + } + } + + match query_type { + QueryType::CompareDateRangeQuery => { + row.push(get_date_range_value(time_dimensions)?); + } + QueryType::BlendingQuery => { + let blending_key = get_blending_response_key(time_dimensions)?; + + if let Some(alias) = members_to_alias_map.get(&blending_key) { + if let Some(key) = columns_pos.get(alias) { + if let Some(value) = db_row.get(*key) { + let member_type = annotation.get(alias).map_or("", |annotation_item| { + annotation_item.member_type.as_deref().unwrap_or("") + }); + + row.push(transform_value(value.clone(), member_type)); + } + } + } + } + _ => {} + } + + Ok(row) +} + +/// Convert DB response object to the vanilla output format. +pub fn get_vanilla_row( + alias_to_member_name_map: &HashMap, + annotation: &HashMap, + query_type: &QueryType, + query: &NormalizedQuery, + db_row: &[DBResponseValue], + columns_pos: &HashMap, +) -> Result> { + let mut row = HashMap::new(); + + for (alias, &index) in columns_pos { + if let Some(value) = db_row.get(index) { + let member_name = match alias_to_member_name_map.get(alias) { + Some(m) => m, + None => { + bail!("Missing member name for alias: {}", alias); + } + }; + + let annotation_for_member = match annotation.get(member_name) { + Some(am) => am, + None => { + bail!( + concat!( + "You requested hidden member: '{}'. Please make it visible using `shown: true`. ", + "Please note primaryKey fields are `shown: false` by default: ", + "https://cube.dev/docs/schema/reference/joins#setting-a-primary-key." + ), + alias + ) + } + }; + + let transformed_value = transform_value( + value.clone(), + annotation_for_member + .member_type + .as_ref() + .unwrap_or(&"".to_string()), + ); + + row.insert(member_name.clone(), transformed_value.clone()); + + // Handle deprecated time dimensions without granularity + let path: Vec<&str> = member_name.split(MEMBER_SEPARATOR).collect(); + let member_name_without_granularity = + format!("{}{}{}", path[0], MEMBER_SEPARATOR, path[1]); + if path.len() == 3 + && query.dimensions.as_ref().map_or(true, |dims| { + !dims.iter().any(|dim| { + *dim == MemberOrMemberExpression::Member( + member_name_without_granularity.clone(), + ) + }) + }) + { + row.insert(member_name_without_granularity, transformed_value); + } + } + } + + match query_type { + QueryType::CompareDateRangeQuery => { + let date_range_value = get_date_range_value(query.time_dimensions.as_ref())?; + row.insert("compareDateRange".to_string(), date_range_value); + } + QueryType::BlendingQuery => { + let blending_key = get_blending_query_key(query.time_dimensions.as_ref())?; + let response_key = get_blending_response_key(query.time_dimensions.as_ref())?; + + if let Some(value) = row.get(&response_key) { + row.insert(blending_key, value.clone()); + } + } + _ => {} + } + + Ok(row) +} + +/// Helper to get a list if unique granularities from normalized queries +pub fn get_query_granularities(queries: &[&NormalizedQuery]) -> Vec { + queries + .iter() + .filter_map(|query| { + query + .time_dimensions + .as_ref() + .and_then(|tds| tds.first()) + .and_then(|td| td.granularity.clone()) + }) + .collect::>() + .into_iter() + .collect() +} + +/// Get Pivot Query for a list of queries +pub fn get_pivot_query( + query_type: &QueryType, + queries: &Vec<&NormalizedQuery>, +) -> Result { + let mut pivot_query = queries + .first() + .copied() + .cloned() + .ok_or_else(|| anyhow::anyhow!("Queries list cannot be empty"))?; + + match query_type { + QueryType::BlendingQuery => { + // Merge and deduplicate measures and dimensions across all queries + let mut merged_measures = HashSet::new(); + let mut merged_dimensions = HashSet::new(); + + for query in queries { + if let Some(measures) = &query.measures { + merged_measures.extend(measures.iter().cloned()); + } + if let Some(dimensions) = &query.dimensions { + merged_dimensions.extend(dimensions.iter().cloned()); + } + } + + pivot_query.measures = if !merged_measures.is_empty() { + Some(merged_measures.into_iter().collect()) + } else { + None + }; + pivot_query.dimensions = if !merged_dimensions.is_empty() { + Some(merged_dimensions.into_iter().collect()) + } else { + None + }; + + // Add time dimensions + let granularities = get_query_granularities(queries); + if !granularities.is_empty() { + pivot_query.time_dimensions = Some(vec![QueryTimeDimension { + dimension: "time".to_string(), + date_range: None, + compare_date_range: None, + granularity: granularities.first().cloned(), + }]); + } + } + QueryType::CompareDateRangeQuery => { + let mut dimensions = vec![MemberOrMemberExpression::Member( + "compareDateRange".to_string(), + )]; + if let Some(dims) = pivot_query.dimensions { + dimensions.extend(dims.clone()); + } + pivot_query.dimensions = Option::from(dimensions); + } + _ => {} + } + + pivot_query.query_type = Option::from(query_type.clone()); + + Ok(pivot_query) +} + +pub fn get_final_cubestore_result_array( + transform_requests: &[TransformDataRequest], + cube_store_results: &[Arc], + result_data: &mut [RequestResultData], +) -> Result<()> { + for (transform_data, cube_store_result, result) in multizip(( + transform_requests.iter(), + cube_store_results.iter(), + result_data.iter_mut(), + )) { + result.prepare_results(transform_data, cube_store_result)?; + } + + Ok(()) +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(untagged)] +pub enum TransformedData { + Compact { + members: Vec, + dataset: Vec>, + }, + Vanilla(Vec>), +} + +impl TransformedData { + /// Transforms queried data array to the output format. + pub fn transform( + request_data: &TransformDataRequest, + cube_store_result: &QueryResult, + ) -> Result { + let alias_to_member_name_map = &request_data.alias_to_member_name_map; + let annotation = &request_data.annotation; + let query = &request_data.query; + let query_type = &request_data.query_type.clone().unwrap_or_default(); + let res_type = request_data.res_type.clone(); + + let (members_to_alias_map, members) = get_members( + query_type, + query, + cube_store_result, + alias_to_member_name_map, + annotation, + )?; + + match res_type { + Some(ResultType::Compact) => { + let dataset: Vec<_> = cube_store_result + .rows + .iter() + .map(|row| { + get_compact_row( + &members_to_alias_map, + annotation, + query_type, + &members, + query.time_dimensions.as_ref(), + row, + &cube_store_result.columns_pos, + ) + }) + .collect::>>()?; + Ok(TransformedData::Compact { members, dataset }) + } + _ => { + let dataset: Vec<_> = cube_store_result + .rows + .iter() + .map(|row| { + get_vanilla_row( + alias_to_member_name_map, + annotation, + query_type, + query, + row, + &cube_store_result.columns_pos, + ) + }) + .collect::>>()?; + Ok(TransformedData::Vanilla(dataset)) + } + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RequestResultDataMulti { + pub query_type: QueryType, + pub results: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub pivot_query: Option, + pub slow_query: bool, +} + +impl RequestResultDataMulti { + /// Processes multiple results and populates the final `RequestResultDataMulti` structure + /// which is sent to the client. + pub fn prepare_results( + &mut self, + request_data: &[TransformDataRequest], + cube_store_result: &[Arc], + ) -> Result<()> { + for (transform_data, cube_store_result, result) in multizip(( + request_data.iter(), + cube_store_result.iter(), + self.results.iter_mut(), + )) { + result.prepare_results(transform_data, cube_store_result)?; + } + + let normalized_queries = self + .results + .iter() + .map(|result| &result.query) + .collect::>(); + + self.pivot_query = Some(get_pivot_query(&self.query_type, &normalized_queries)?); + + Ok(()) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RequestResultData { + pub query: NormalizedQuery, + #[serde(skip_serializing_if = "Option::is_none")] + pub last_refresh_time: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub refresh_key_values: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub used_pre_aggregations: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub transformed_query: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub request_id: Option, + pub annotation: HashMap>, + #[serde(skip_serializing_if = "Option::is_none")] + pub data_source: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub db_type: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ext_db_type: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub external: Option, + pub slow_query: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub total: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub data: Option, +} + +impl RequestResultData { + /// Populates the `RequestResultData` structure with the transformed Query result. + pub fn prepare_results( + &mut self, + request_data: &TransformDataRequest, + cube_store_result: &QueryResult, + ) -> Result<()> { + let transformed = TransformedData::transform(request_data, cube_store_result)?; + self.data = Some(transformed); + + Ok(()) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RequestResultArray { + pub results: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(untagged)] +pub enum DBResponsePrimitive { + Null, + Boolean(bool), + Number(f64), + String(String), +} + +impl Display for DBResponsePrimitive { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let str = match self { + DBResponsePrimitive::Null => "null".to_string(), + DBResponsePrimitive::Boolean(b) => b.to_string(), + DBResponsePrimitive::Number(n) => n.to_string(), + DBResponsePrimitive::String(s) => s.clone(), + }; + write!(f, "{}", str) + } +} + +#[derive(Debug, Clone, Deserialize)] +pub enum DBResponseValue { + DateTime(DateTime), + Primitive(DBResponsePrimitive), + // TODO: Is this variant still used? + Object { value: DBResponsePrimitive }, +} + +impl Display for DBResponseValue { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let str = match self { + DBResponseValue::DateTime(dt) => dt.to_rfc3339(), + DBResponseValue::Primitive(p) => p.to_string(), + DBResponseValue::Object { value } => value.to_string(), + }; + write!(f, "{}", str) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::transport::JsRawData; + use anyhow::Result; + use chrono::{TimeZone, Timelike, Utc}; + use serde_json::from_str; + use std::{fmt, sync::LazyLock}; + + type TestSuiteData = HashMap; + + #[derive(Clone, Deserialize)] + #[serde(rename_all = "camelCase")] + struct TestData { + request: TransformDataRequest, + query_result: JsRawData, + final_result_default: Option, + final_result_compact: Option, + } + + const TEST_SUITE_JSON: &str = r#" +{ + "regular_discount_by_city": { + "request": { + "aliasToMemberNameMap": { + "e_commerce_records_us2021__avg_discount": "ECommerceRecordsUs2021.avg_discount", + "e_commerce_records_us2021__city": "ECommerceRecordsUs2021.city" + }, + "annotation": { + "ECommerceRecordsUs2021.avg_discount": { + "title": "E Commerce Records Us2021 Avg Discount", + "shortTitle": "Avg Discount", + "type": "number", + "drillMembers": [], + "drillMembersGrouped": { + "measures": [], + "dimensions": [] + } + }, + "ECommerceRecordsUs2021.city": { + "title": "E Commerce Records Us2021 City", + "shortTitle": "City", + "type": "string" + } + }, + "query": { + "dimensions": [ + "ECommerceRecordsUs2021.city" + ], + "measures": [ + "ECommerceRecordsUs2021.avg_discount" + ], + "limit": 2, + "rowLimit": 2, + "timezone": "UTC", + "order": [], + "filters": [], + "timeDimensions": [] + }, + "queryType": "regularQuery" + }, + "queryResult": [ + { + "e_commerce_records_us2021__city": "Missouri City", + "e_commerce_records_us2021__avg_discount": "0.80000000000000000000" + }, + { + "e_commerce_records_us2021__city": "Abilene", + "e_commerce_records_us2021__avg_discount": "0.80000000000000000000" + } + ], + "finalResultDefault": [ + { + "ECommerceRecordsUs2021.city": "Missouri City", + "ECommerceRecordsUs2021.avg_discount": "0.80000000000000000000" + }, + { + "ECommerceRecordsUs2021.city": "Abilene", + "ECommerceRecordsUs2021.avg_discount": "0.80000000000000000000" + } + ], + "finalResultCompact": { + "members": [ + "ECommerceRecordsUs2021.city", + "ECommerceRecordsUs2021.avg_discount" + ], + "dataset": [ + [ + "Missouri City", + "0.80000000000000000000" + ], + [ + "Abilene", + "0.80000000000000000000" + ] + ] + } + }, + "regular_profit_by_postal_code": { + "request": { + "aliasToMemberNameMap": { + "e_commerce_records_us2021__avg_profit": "ECommerceRecordsUs2021.avg_profit", + "e_commerce_records_us2021__postal_code": "ECommerceRecordsUs2021.postalCode" + }, + "annotation": { + "ECommerceRecordsUs2021.avg_profit": { + "title": "E Commerce Records Us2021 Avg Profit", + "shortTitle": "Avg Profit", + "type": "number", + "drillMembers": [], + "drillMembersGrouped": { + "measures": [], + "dimensions": [] + } + }, + "ECommerceRecordsUs2021.postalCode": { + "title": "E Commerce Records Us2021 Postal Code", + "shortTitle": "Postal Code", + "type": "string" + } + }, + "query": { + "dimensions": [ + "ECommerceRecordsUs2021.postalCode" + ], + "measures": [ + "ECommerceRecordsUs2021.avg_profit" + ], + "limit": 2, + "rowLimit": 2, + "timezone": "UTC", + "order": [], + "filters": [], + "timeDimensions": [] + }, + "queryType": "regularQuery" + }, + "queryResult": [ + { + "e_commerce_records_us2021__postal_code": "95823", + "e_commerce_records_us2021__avg_profit": "646.1258666666666667" + }, + { + "e_commerce_records_us2021__postal_code": "64055", + "e_commerce_records_us2021__avg_profit": "487.8315000000000000" + } + ], + "finalResultDefault": [ + { + "ECommerceRecordsUs2021.postalCode": "95823", + "ECommerceRecordsUs2021.avg_profit": "646.1258666666666667" + }, + { + "ECommerceRecordsUs2021.postalCode": "64055", + "ECommerceRecordsUs2021.avg_profit": "487.8315000000000000" + } + ], + "finalResultCompact": { + "members": [ + "ECommerceRecordsUs2021.postalCode", + "ECommerceRecordsUs2021.avg_profit" + ], + "dataset": [ + [ + "95823", + "646.1258666666666667" + ], + [ + "64055", + "487.8315000000000000" + ] + ] + } + }, + "compare_date_range_count_by_order_date": { + "request": { + "aliasToMemberNameMap": { + "e_commerce_records_us2021__count": "ECommerceRecordsUs2021.count", + "e_commerce_records_us2021__order_date_day": "ECommerceRecordsUs2021.orderDate.day" + }, + "annotation": { + "ECommerceRecordsUs2021.count": { + "title": "E Commerce Records Us2021 Count", + "shortTitle": "Count", + "type": "number", + "drillMembers": [ + "ECommerceRecordsUs2021.city", + "ECommerceRecordsUs2021.country", + "ECommerceRecordsUs2021.customerId", + "ECommerceRecordsUs2021.orderId", + "ECommerceRecordsUs2021.productId", + "ECommerceRecordsUs2021.productName", + "ECommerceRecordsUs2021.orderDate" + ], + "drillMembersGrouped": { + "measures": [], + "dimensions": [ + "ECommerceRecordsUs2021.city", + "ECommerceRecordsUs2021.country", + "ECommerceRecordsUs2021.customerId", + "ECommerceRecordsUs2021.orderId", + "ECommerceRecordsUs2021.productId", + "ECommerceRecordsUs2021.productName", + "ECommerceRecordsUs2021.orderDate" + ] + } + }, + "ECommerceRecordsUs2021.orderDate.day": { + "title": "E Commerce Records Us2021 Order Date", + "shortTitle": "Order Date", + "type": "time" + }, + "ECommerceRecordsUs2021.orderDate": { + "title": "E Commerce Records Us2021 Order Date", + "shortTitle": "Order Date", + "type": "time" + } + }, + "query": { + "measures": [ + "ECommerceRecordsUs2021.count" + ], + "timeDimensions": [ + { + "dimension": "ECommerceRecordsUs2021.orderDate", + "granularity": "day", + "dateRange": [ + "2020-01-01T00:00:00.000", + "2020-01-31T23:59:59.999" + ] + } + ], + "limit": 2, + "rowLimit": 2, + "timezone": "UTC", + "order": [], + "filters": [], + "dimensions": [] + }, + "queryType": "compareDateRangeQuery" + }, + "queryResult": [ + { + "e_commerce_records_us2021__order_date_day": "2020-01-01T00:00:00.000", + "e_commerce_records_us2021__count": "10" + }, + { + "e_commerce_records_us2021__order_date_day": "2020-01-02T00:00:00.000", + "e_commerce_records_us2021__count": "8" + } + ], + "finalResultDefault": [ + { + "ECommerceRecordsUs2021.orderDate.day": "2020-01-01T00:00:00.000", + "ECommerceRecordsUs2021.orderDate": "2020-01-01T00:00:00.000", + "ECommerceRecordsUs2021.count": "10", + "compareDateRange": "2020-01-01T00:00:00.000 - 2020-01-31T23:59:59.999" + }, + { + "ECommerceRecordsUs2021.orderDate.day": "2020-01-02T00:00:00.000", + "ECommerceRecordsUs2021.orderDate": "2020-01-02T00:00:00.000", + "ECommerceRecordsUs2021.count": "8", + "compareDateRange": "2020-01-01T00:00:00.000 - 2020-01-31T23:59:59.999" + } + ], + "finalResultCompact": { + "members": [ + "ECommerceRecordsUs2021.orderDate.day", + "ECommerceRecordsUs2021.orderDate", + "ECommerceRecordsUs2021.count", + "compareDateRange" + ], + "dataset": [ + [ + "2020-01-01T00:00:00.000", + "2020-01-01T00:00:00.000", + "10", + "2020-01-01T00:00:00.000 - 2020-01-31T23:59:59.999" + ], + [ + "2020-01-02T00:00:00.000", + "2020-01-02T00:00:00.000", + "8", + "2020-01-01T00:00:00.000 - 2020-01-31T23:59:59.999" + ] + ] + } + }, + "compare_date_range_count_by_order_date2": { + "request": { + "aliasToMemberNameMap": { + "e_commerce_records_us2021__count": "ECommerceRecordsUs2021.count", + "e_commerce_records_us2021__order_date_day": "ECommerceRecordsUs2021.orderDate.day" + }, + "annotation": { + "ECommerceRecordsUs2021.count": { + "title": "E Commerce Records Us2021 Count", + "shortTitle": "Count", + "type": "number", + "drillMembers": [ + "ECommerceRecordsUs2021.city", + "ECommerceRecordsUs2021.country", + "ECommerceRecordsUs2021.customerId", + "ECommerceRecordsUs2021.orderId", + "ECommerceRecordsUs2021.productId", + "ECommerceRecordsUs2021.productName", + "ECommerceRecordsUs2021.orderDate" + ], + "drillMembersGrouped": { + "measures": [], + "dimensions": [ + "ECommerceRecordsUs2021.city", + "ECommerceRecordsUs2021.country", + "ECommerceRecordsUs2021.customerId", + "ECommerceRecordsUs2021.orderId", + "ECommerceRecordsUs2021.productId", + "ECommerceRecordsUs2021.productName", + "ECommerceRecordsUs2021.orderDate" + ] + } + }, + "ECommerceRecordsUs2021.orderDate.day": { + "title": "E Commerce Records Us2021 Order Date", + "shortTitle": "Order Date", + "type": "time" + }, + "ECommerceRecordsUs2021.orderDate": { + "title": "E Commerce Records Us2021 Order Date", + "shortTitle": "Order Date", + "type": "time" + } + }, + "query": { + "measures": [ + "ECommerceRecordsUs2021.count" + ], + "timeDimensions": [ + { + "dimension": "ECommerceRecordsUs2021.orderDate", + "granularity": "day", + "dateRange": [ + "2020-03-01T00:00:00.000", + "2020-03-31T23:59:59.999" + ] + } + ], + "limit": 2, + "rowLimit": 2, + "timezone": "UTC", + "order": [], + "filters": [], + "dimensions": [] + }, + "queryType": "compareDateRangeQuery" + }, + "queryResult": [ + { + "e_commerce_records_us2021__order_date_day": "2020-03-02T00:00:00.000", + "e_commerce_records_us2021__count": "11" + }, + { + "e_commerce_records_us2021__order_date_day": "2020-03-03T00:00:00.000", + "e_commerce_records_us2021__count": "7" + } + ], + "finalResultDefault": [ + { + "ECommerceRecordsUs2021.orderDate.day": "2020-03-02T00:00:00.000", + "ECommerceRecordsUs2021.orderDate": "2020-03-02T00:00:00.000", + "ECommerceRecordsUs2021.count": "11", + "compareDateRange": "2020-03-01T00:00:00.000 - 2020-03-31T23:59:59.999" + }, + { + "ECommerceRecordsUs2021.orderDate.day": "2020-03-03T00:00:00.000", + "ECommerceRecordsUs2021.orderDate": "2020-03-03T00:00:00.000", + "ECommerceRecordsUs2021.count": "7", + "compareDateRange": "2020-03-01T00:00:00.000 - 2020-03-31T23:59:59.999" + } + ], + "finalResultCompact": { + "members": [ + "ECommerceRecordsUs2021.orderDate.day", + "ECommerceRecordsUs2021.orderDate", + "ECommerceRecordsUs2021.count", + "compareDateRange" + ], + "dataset": [ + [ + "2020-03-02T00:00:00.000", + "2020-03-02T00:00:00.000", + "11", + "2020-03-01T00:00:00.000 - 2020-03-31T23:59:59.999" + ], + [ + "2020-03-03T00:00:00.000", + "2020-03-03T00:00:00.000", + "7", + "2020-03-01T00:00:00.000 - 2020-03-31T23:59:59.999" + ] + ] + } + }, + "blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode": { + "request": { + "aliasToMemberNameMap": { + "e_commerce_records_us2021__avg_discount": "ECommerceRecordsUs2021.avg_discount", + "e_commerce_records_us2021__order_date_month": "ECommerceRecordsUs2021.orderDate.month" + }, + "annotation": { + "ECommerceRecordsUs2021.avg_discount": { + "title": "E Commerce Records Us2021 Avg Discount", + "shortTitle": "Avg Discount", + "type": "number", + "drillMembers": [], + "drillMembersGrouped": { + "measures": [], + "dimensions": [] + } + }, + "ECommerceRecordsUs2021.orderDate.month": { + "title": "E Commerce Records Us2021 Order Date", + "shortTitle": "Order Date", + "type": "time" + }, + "ECommerceRecordsUs2021.orderDate": { + "title": "E Commerce Records Us2021 Order Date", + "shortTitle": "Order Date", + "type": "time" + } + }, + "query": { + "measures": [ + "ECommerceRecordsUs2021.avg_discount" + ], + "timeDimensions": [ + { + "dimension": "ECommerceRecordsUs2021.orderDate", + "granularity": "month", + "dateRange": [ + "2020-01-01T00:00:00.000", + "2020-12-30T23:59:59.999" + ] + } + ], + "filters": [ + { + "operator": "equals", + "values": [ + "Standard Class" + ], + "member": "ECommerceRecordsUs2021.shipMode" + } + ], + "limit": 2, + "rowLimit": 2, + "timezone": "UTC", + "order": [], + "dimensions": [] + }, + "queryType": "blendingQuery" + }, + "queryResult": [ + { + "e_commerce_records_us2021__order_date_month": "2020-01-01T00:00:00.000", + "e_commerce_records_us2021__avg_discount": "0.15638297872340425532" + }, + { + "e_commerce_records_us2021__order_date_month": "2020-02-01T00:00:00.000", + "e_commerce_records_us2021__avg_discount": "0.17573529411764705882" + } + ], + "finalResultDefault": [ + { + "ECommerceRecordsUs2021.orderDate.month": "2020-01-01T00:00:00.000", + "ECommerceRecordsUs2021.orderDate": "2020-01-01T00:00:00.000", + "ECommerceRecordsUs2021.avg_discount": "0.15638297872340425532", + "time.month": "2020-01-01T00:00:00.000" + }, + { + "ECommerceRecordsUs2021.orderDate.month": "2020-02-01T00:00:00.000", + "ECommerceRecordsUs2021.orderDate": "2020-02-01T00:00:00.000", + "ECommerceRecordsUs2021.avg_discount": "0.17573529411764705882", + "time.month": "2020-02-01T00:00:00.000" + } + ], + "finalResultCompact": { + "members": [ + "ECommerceRecordsUs2021.orderDate.month", + "ECommerceRecordsUs2021.orderDate", + "ECommerceRecordsUs2021.avg_discount", + "time.month" + ], + "dataset": [ + [ + "2020-01-01T00:00:00.000", + "2020-01-01T00:00:00.000", + "0.15638297872340425532", + "2020-01-01T00:00:00.000" + ], + [ + "2020-02-01T00:00:00.000", + "2020-02-01T00:00:00.000", + "0.17573529411764705882", + "2020-02-01T00:00:00.000" + ] + ] + } + }, + "blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode2": { + "request": { + "aliasToMemberNameMap": { + "e_commerce_records_us2021__avg_discount": "ECommerceRecordsUs2021.avg_discount", + "e_commerce_records_us2021__order_date_month": "ECommerceRecordsUs2021.orderDate.month" + }, + "annotation": { + "ECommerceRecordsUs2021.avg_discount": { + "title": "E Commerce Records Us2021 Avg Discount", + "shortTitle": "Avg Discount", + "type": "number", + "drillMembers": [], + "drillMembersGrouped": { + "measures": [], + "dimensions": [] + } + }, + "ECommerceRecordsUs2021.orderDate.month": { + "title": "E Commerce Records Us2021 Order Date", + "shortTitle": "Order Date", + "type": "time" + }, + "ECommerceRecordsUs2021.orderDate": { + "title": "E Commerce Records Us2021 Order Date", + "shortTitle": "Order Date", + "type": "time" + } + }, + "query": { + "measures": [ + "ECommerceRecordsUs2021.avg_discount" + ], + "timeDimensions": [ + { + "dimension": "ECommerceRecordsUs2021.orderDate", + "granularity": "month", + "dateRange": [ + "2020-01-01T00:00:00.000", + "2020-12-30T23:59:59.999" + ] + } + ], + "filters": [ + { + "operator": "equals", + "values": [ + "First Class" + ], + "member": "ECommerceRecordsUs2021.shipMode" + } + ], + "limit": 2, + "rowLimit": 2, + "timezone": "UTC", + "order": [], + "dimensions": [] + }, + "queryType": "blendingQuery" + }, + "queryResult": [ + { + "e_commerce_records_us2021__order_date_month": "2020-01-01T00:00:00.000", + "e_commerce_records_us2021__avg_discount": "0.28571428571428571429" + }, + { + "e_commerce_records_us2021__order_date_month": "2020-02-01T00:00:00.000", + "e_commerce_records_us2021__avg_discount": "0.21777777777777777778" + } + ], + "finalResultDefault": [ + { + "ECommerceRecordsUs2021.orderDate.month": "2020-01-01T00:00:00.000", + "ECommerceRecordsUs2021.orderDate": "2020-01-01T00:00:00.000", + "ECommerceRecordsUs2021.avg_discount": "0.28571428571428571429", + "time.month": "2020-01-01T00:00:00.000" + }, + { + "ECommerceRecordsUs2021.orderDate.month": "2020-02-01T00:00:00.000", + "ECommerceRecordsUs2021.orderDate": "2020-02-01T00:00:00.000", + "ECommerceRecordsUs2021.avg_discount": "0.21777777777777777778", + "time.month": "2020-02-01T00:00:00.000" + } + ], + "finalResultCompact": { + "members": [ + "ECommerceRecordsUs2021.orderDate.month", + "ECommerceRecordsUs2021.orderDate", + "ECommerceRecordsUs2021.avg_discount", + "time.month" + ], + "dataset": [ + [ + "2020-01-01T00:00:00.000", + "2020-01-01T00:00:00.000", + "0.28571428571428571429", + "2020-01-01T00:00:00.000" + ], + [ + "2020-02-01T00:00:00.000", + "2020-02-01T00:00:00.000", + "0.21777777777777777778", + "2020-02-01T00:00:00.000" + ] + ] + } + } +} + "#; + + static TEST_SUITE_DATA: LazyLock = + LazyLock::new(|| from_str(TEST_SUITE_JSON).unwrap()); + + #[derive(Debug)] + pub struct TestError(String); + + impl Display for TestError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Error: {}", self.0) + } + } + + impl std::error::Error for TestError {} + + /// Smart comparator of datasets. + /// Hashmaps don't guarantee the order of the elements while iterating, + /// so it's not possible to simply compare generated one and the one from the json. + fn compare_transformed_data( + left: &TransformedData, + right: &TransformedData, + ) -> Result<(), TestError> { + match (left, right) { + ( + TransformedData::Compact { + members: left_members, + dataset: left_dataset, + }, + TransformedData::Compact { + members: right_members, + dataset: right_dataset, + }, + ) => { + let mut left_sorted_members = left_members.clone(); + let mut right_sorted_members = right_members.clone(); + left_sorted_members.sort(); + right_sorted_members.sort(); + + if left_sorted_members != right_sorted_members { + return Err(TestError("Members do not match after sorting".to_string())); + } + + if left_dataset.len() != right_dataset.len() { + return Err(TestError("Datasets have different lengths".to_string())); + } + + let mut member_index_map = HashMap::new(); + for (i, member) in left_members.iter().enumerate() { + if let Some(right_index) = right_members.iter().position(|x| x == member) { + member_index_map.insert(i, right_index); + } else { + return Err(TestError("Member not found in right object".to_string())); + } + } + + for (i, left_row) in left_dataset.iter().enumerate() { + let right_row = &right_dataset[i]; + + for (j, left_value) in left_row.iter().enumerate() { + let mapped_index = *member_index_map.get(&j).unwrap(); + let right_value = &right_row[mapped_index]; + if left_value != right_value { + return Err(TestError(format!( + "Dataset values at row {} and column {} do not match: {} != {}", + i, j, left_value, right_value + ))); + } + } + } + + Ok(()) + } + (TransformedData::Vanilla(left_dataset), TransformedData::Vanilla(right_dataset)) => { + if left_dataset.len() != right_dataset.len() { + return Err(TestError( + "Vanilla datasets have different lengths".to_string(), + )); + } + + for (i, (left_record, right_record)) in + left_dataset.iter().zip(right_dataset.iter()).enumerate() + { + if left_record.len() != right_record.len() { + return Err(TestError(format!( + "Vanilla dataset records at index {} have different numbers of keys", + i + ))); + } + + for (key, left_value) in left_record { + if let Some(right_value) = right_record.get(key) { + if left_value != right_value { + return Err(TestError(format!( + "Values at index {} for key '{}' do not match: {:?} != {:?}", + i, key, left_value, right_value + ))); + } + } else { + return Err(TestError(format!( + "Key '{}' not found in right record at index {}", + key, i + ))); + } + } + } + + Ok(()) + } + _ => Err(TestError("Mismatched TransformedData types".to_string())), + } + } + + #[test] + fn test_transform_value_datetime_to_time() { + let dt = Utc + .with_ymd_and_hms(2024, 1, 1, 12, 30, 15) + .unwrap() + .with_nanosecond(123_000_000) + .unwrap(); + let value = DBResponseValue::DateTime(dt); + let result = transform_value(value, "time"); + + assert_eq!( + result, + DBResponsePrimitive::String("2024-01-01T12:30:15.123".to_string()) + ); + } + + #[test] + fn test_transform_value_datetime_empty_type() { + let dt = Utc + .with_ymd_and_hms(2024, 1, 1, 12, 30, 15) + .unwrap() + .with_nanosecond(123_000_000) + .unwrap(); + let value = DBResponseValue::DateTime(dt); + let result = transform_value(value, ""); + + assert_eq!( + result, + DBResponsePrimitive::String("2024-01-01T12:30:15.123".to_string()) + ); + } + + #[test] + fn test_transform_value_string_to_time_valid_rfc3339() { + let value = DBResponseValue::Primitive(DBResponsePrimitive::String( + "2024-01-01T12:30:15.123".to_string(), + )); + let result = transform_value(value, "time"); + + assert_eq!( + result, + DBResponsePrimitive::String("2024-01-01T12:30:15.123".to_string()) + ); + } + + #[test] + fn test_transform_value_string_wo_t_to_time_valid_rfc3339() { + let value = DBResponseValue::Primitive(DBResponsePrimitive::String( + "2024-01-01 12:30:15.123".to_string(), + )); + let result = transform_value(value, "time"); + + assert_eq!( + result, + DBResponsePrimitive::String("2024-01-01T12:30:15.123".to_string()) + ); + } + + #[test] + fn test_transform_value_string_wo_mssec_to_time_valid_rfc3339() { + let value = DBResponseValue::Primitive(DBResponsePrimitive::String( + "2024-01-01 12:30:15".to_string(), + )); + let result = transform_value(value, "time"); + + assert_eq!( + result, + DBResponsePrimitive::String("2024-01-01T12:30:15.000".to_string()) + ); + } + + #[test] + fn test_transform_value_string_wo_mssec_w_t_to_time_valid_rfc3339() { + let value = DBResponseValue::Primitive(DBResponsePrimitive::String( + "2024-01-01T12:30:15".to_string(), + )); + let result = transform_value(value, "time"); + + assert_eq!( + result, + DBResponsePrimitive::String("2024-01-01T12:30:15.000".to_string()) + ); + } + + #[test] + fn test_transform_value_string_with_tz_offset_to_time_valid_rfc3339() { + let value = DBResponseValue::Primitive(DBResponsePrimitive::String( + "2024-01-01 12:30:15.123 +00:00".to_string(), + )); + let result = transform_value(value, "time"); + + assert_eq!( + result, + DBResponsePrimitive::String("2024-01-01T12:30:15.123".to_string()) + ); + } + + #[test] + fn test_transform_value_string_with_tz_to_time_valid_rfc3339() { + let value = DBResponseValue::Primitive(DBResponsePrimitive::String( + "2024-01-01 12:30:15.123 UTC".to_string(), + )); + let result = transform_value(value, "time"); + + assert_eq!( + result, + DBResponsePrimitive::String("2024-01-01T12:30:15.123".to_string()) + ); + } + + #[test] + fn test_transform_value_string_to_time_invalid_rfc3339() { + let value = + DBResponseValue::Primitive(DBResponsePrimitive::String("invalid-date".to_string())); + let result = transform_value(value, "time"); + + assert_eq!( + result, + DBResponsePrimitive::String("invalid-date".to_string()) + ); + } + + #[test] + fn test_transform_value_primitive_string_type_not_time() { + let value = + DBResponseValue::Primitive(DBResponsePrimitive::String("some-string".to_string())); + let result = transform_value(value, "other"); + + assert_eq!( + result, + DBResponsePrimitive::String("some-string".to_string()) + ); + } + + #[test] + fn test_transform_value_object() { + let obj_value = DBResponsePrimitive::String("object-value".to_string()); + let value = DBResponseValue::Object { + value: obj_value.clone(), + }; + let result = transform_value(value, "time"); + + assert_eq!(result, obj_value); + } + + #[test] + fn test_transform_value_fallback_to_null() { + let value = DBResponseValue::DateTime(Utc::now()); + let result = transform_value(value, "unknown"); + + assert_eq!(result, DBResponsePrimitive::Null); + } + + #[test] + fn test_get_date_range_value_valid_range() -> Result<()> { + let time_dimensions = vec![QueryTimeDimension { + dimension: "some-dim".to_string(), + date_range: Some(vec![ + "2024-01-01T00:00:00Z".to_string(), + "2024-01-31T23:59:59Z".to_string(), + ]), + compare_date_range: None, + granularity: None, + }]; + + let result = get_date_range_value(Some(&time_dimensions))?; + assert_eq!( + result, + DBResponsePrimitive::String("2024-01-01T00:00:00Z - 2024-01-31T23:59:59Z".to_string()) + ); + Ok(()) + } + + #[test] + fn test_get_date_range_value_no_time_dimensions() { + let result = get_date_range_value(None); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "QueryTimeDimension should be specified for the compare date range query." + ); + } + + #[test] + fn test_get_date_range_value_empty_time_dimensions() { + let time_dimensions: Vec = vec![]; + + let result = get_date_range_value(Some(&time_dimensions)); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "No time dimension provided." + ); + } + + #[test] + fn test_get_date_range_value_missing_date_range() { + let time_dimensions = vec![QueryTimeDimension { + dimension: "dim".to_string(), + date_range: None, + compare_date_range: None, + granularity: None, + }]; + + let result = get_date_range_value(Some(&time_dimensions)); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "Inconsistent QueryTimeDimension configuration: dateRange required." + ); + } + + #[test] + fn test_get_date_range_value_single_date_range() { + let time_dimensions = vec![QueryTimeDimension { + dimension: "dim".to_string(), + date_range: Some(vec!["2024-01-01T00:00:00Z".to_string()]), + compare_date_range: None, + granularity: None, + }]; + + let result = get_date_range_value(Some(&time_dimensions)); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "Inconsistent dateRange configuration for the compare date range query: 2024-01-01T00:00:00Z" + ); + } + + #[test] + fn test_get_blending_query_key_valid_granularity() -> Result<()> { + let time_dimensions = vec![QueryTimeDimension { + dimension: "dim".to_string(), + granularity: Some("day".to_string()), + date_range: None, + compare_date_range: None, + }]; + + let result = get_blending_query_key(Some(&time_dimensions))?; + assert_eq!(result, "time.day"); + Ok(()) + } + + #[test] + fn test_get_blending_query_key_no_time_dimensions() { + let result = get_blending_query_key(None); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "QueryTimeDimension should be specified for the blending query." + ); + } + + #[test] + fn test_get_blending_query_key_empty_time_dimensions() { + let time_dimensions: Vec = vec![]; + + let result = get_blending_query_key(Some(&time_dimensions)); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "QueryTimeDimension should be specified for the blending query." + ); + } + + #[test] + fn test_get_blending_query_key_missing_granularity() { + let time_dimensions = vec![QueryTimeDimension { + dimension: "dim".to_string(), + granularity: None, + date_range: None, + compare_date_range: None, + }]; + + let result = get_blending_query_key(Some(&time_dimensions)); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + format!( + "Inconsistent QueryTimeDimension configuration for the blending query, granularity required: {:?}", + QueryTimeDimension { + dimension: "dim".to_string(), + granularity: None, + date_range: None, + compare_date_range: None, + } + ) + ); + } + + #[test] + fn test_get_blending_response_key_valid_dimension_and_granularity() -> Result<()> { + let time_dimensions = vec![QueryTimeDimension { + dimension: "orders.created_at".to_string(), + granularity: Some("day".to_string()), + date_range: None, + compare_date_range: None, + }]; + + let result = get_blending_response_key(Some(&time_dimensions))?; + assert_eq!(result, "orders.created_at.day"); + Ok(()) + } + + #[test] + fn test_get_blending_response_key_no_time_dimensions() { + let result = get_blending_response_key(None); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "QueryTimeDimension should be specified for the blending query." + ); + } + + #[test] + fn test_get_blending_response_key_empty_time_dimensions() { + let time_dimensions: Vec = vec![]; + + let result = get_blending_response_key(Some(&time_dimensions)); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "QueryTimeDimension should be specified for the blending query." + ); + } + + #[test] + fn test_get_blending_response_key_missing_granularity() { + let time_dimensions = vec![QueryTimeDimension { + dimension: "orders.created_at".to_string(), + granularity: None, + date_range: None, + compare_date_range: None, + }]; + + let result = get_blending_response_key(Some(&time_dimensions)); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + format!( + "Inconsistent QueryTimeDimension configuration for the blending query, granularity required: {:?}", + QueryTimeDimension { + dimension: "orders.created_at".to_string(), + granularity: None, + date_range: None, + compare_date_range: None, + } + ) + ); + } + + #[test] + fn test_regular_profit_by_postal_code_compact() -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get(&"regular_profit_by_postal_code".to_string()) + .unwrap() + .clone(); + test_data.request.res_type = Some(ResultType::Compact); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let transformed = TransformedData::transform(&test_data.request, &raw_data)?; + compare_transformed_data(&transformed, &test_data.final_result_compact.unwrap())?; + Ok(()) + } + + #[test] + fn test_compare_date_range_count_by_order_date() -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get(&"compare_date_range_count_by_order_date".to_string()) + .unwrap() + .clone(); + test_data.request.res_type = Some(ResultType::Compact); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let transformed = TransformedData::transform(&test_data.request, &raw_data)?; + compare_transformed_data(&transformed, &test_data.final_result_compact.unwrap())?; + Ok(()) + } + + #[test] + fn test_compare_date_range_count_by_order_date2() -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get(&"compare_date_range_count_by_order_date2".to_string()) + .unwrap() + .clone(); + test_data.request.res_type = Some(ResultType::Compact); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let transformed = TransformedData::transform(&test_data.request, &raw_data)?; + compare_transformed_data(&transformed, &test_data.final_result_compact.unwrap())?; + Ok(()) + } + + #[test] + fn test_regular_discount_by_city() -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get(&"regular_discount_by_city".to_string()) + .unwrap() + .clone(); + test_data.request.res_type = Some(ResultType::Compact); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let transformed = TransformedData::transform(&test_data.request, &raw_data)?; + compare_transformed_data(&transformed, &test_data.final_result_compact.unwrap())?; + Ok(()) + } + + #[test] + fn test_regular_discount_by_city_to_fail() -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get(&"regular_discount_by_city".to_string()) + .unwrap() + .clone(); + test_data + .request + .alias_to_member_name_map + .remove(&"e_commerce_records_us2021__avg_discount".to_string()); + test_data.request.res_type = Some(ResultType::Compact); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + match TransformedData::transform(&test_data.request, &raw_data) { + Ok(_) => Err(TestError("regular_discount_by_city should fail ".to_string()).into()), + Err(_) => Ok(()), // Should throw an error + } + } + + #[test] + fn test_blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode( + ) -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get( + &"blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode" + .to_string(), + ) + .unwrap() + .clone(); + test_data.request.res_type = Some(ResultType::Compact); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let transformed = TransformedData::transform(&test_data.request, &raw_data)?; + compare_transformed_data(&transformed, &test_data.final_result_compact.unwrap())?; + Ok(()) + } + + #[test] + fn test_blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode2( + ) -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get( + &"blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode" + .to_string(), + ) + .unwrap() + .clone(); + test_data.request.res_type = Some(ResultType::Compact); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let transformed = TransformedData::transform(&test_data.request, &raw_data)?; + compare_transformed_data(&transformed, &test_data.final_result_compact.unwrap())?; + Ok(()) + } + + #[test] + fn test_regular_discount_by_city_default_to_fail() -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get(&"regular_discount_by_city".to_string()) + .unwrap() + .clone(); + test_data + .request + .alias_to_member_name_map + .remove(&"e_commerce_records_us2021__avg_discount".to_string()); + test_data.request.res_type = Some(ResultType::Default); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + match TransformedData::transform(&test_data.request, &raw_data) { + Ok(_) => Err(TestError("regular_discount_by_city should fail ".to_string()).into()), + Err(_) => Ok(()), // Should throw an error + } + } + + #[test] + fn test_regular_discount_by_city_default() -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get(&"regular_discount_by_city".to_string()) + .unwrap() + .clone(); + test_data.request.res_type = Some(ResultType::Default); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let transformed = TransformedData::transform(&test_data.request, &raw_data)?; + compare_transformed_data(&transformed, &test_data.final_result_default.unwrap())?; + Ok(()) + } + + #[test] + fn test_regular_profit_by_postal_code_default() -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get(&"regular_discount_by_city".to_string()) + .unwrap() + .clone(); + test_data.request.res_type = Some(ResultType::Default); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let transformed = TransformedData::transform(&test_data.request, &raw_data)?; + compare_transformed_data(&transformed, &test_data.final_result_default.unwrap())?; + Ok(()) + } + + #[test] + fn test_blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode_default( + ) -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get( + &"blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode" + .to_string(), + ) + .unwrap() + .clone(); + test_data.request.res_type = Some(ResultType::Default); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let transformed = TransformedData::transform(&test_data.request, &raw_data)?; + compare_transformed_data(&transformed, &test_data.final_result_default.unwrap())?; + Ok(()) + } + + #[test] + fn test_blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode2_default( + ) -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get( + &"blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode2" + .to_string(), + ) + .unwrap() + .clone(); + test_data.request.res_type = Some(ResultType::Default); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let transformed = TransformedData::transform(&test_data.request, &raw_data)?; + compare_transformed_data(&transformed, &test_data.final_result_default.unwrap())?; + Ok(()) + } + + #[test] + fn test_get_members_no_alias_to_member_name_map() -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get(&"regular_profit_by_postal_code".to_string()) + .unwrap() + .clone(); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + test_data.request.alias_to_member_name_map = HashMap::new(); + let alias_to_member_name_map = &test_data.request.alias_to_member_name_map; + let annotation = &test_data.request.annotation; + let query = &test_data.request.query; + let query_type = &test_data.request.query_type.clone().unwrap_or_default(); + + match get_members( + query_type, + query, + &raw_data, + alias_to_member_name_map, + annotation, + ) { + Ok(_) => Err(TestError("get_members() should fail ".to_string()).into()), + Err(err) => { + assert!(err.to_string().contains("Member name not found for alias")); + Ok(()) + } + } + } + + #[test] + fn test_get_members_empty_dataset() -> Result<()> { + let test_data = TEST_SUITE_DATA + .get(&"regular_profit_by_postal_code".to_string()) + .unwrap() + .clone(); + let alias_to_member_name_map = &test_data.request.alias_to_member_name_map; + let annotation = &test_data.request.annotation; + let query = &test_data.request.query; + let query_type = &test_data.request.query_type.clone().unwrap_or_default(); + + let (members_to_alias_map, members) = get_members( + query_type, + query, + &QueryResult { + columns: vec![], + rows: vec![], + columns_pos: HashMap::new(), + }, + alias_to_member_name_map, + annotation, + )?; + assert_eq!(members_to_alias_map.len(), 0); + assert_eq!(members.len(), 0); + Ok(()) + } + + #[test] + fn test_get_members_filled_dataset() -> Result<()> { + let test_data = TEST_SUITE_DATA + .get(&"regular_profit_by_postal_code".to_string()) + .unwrap() + .clone(); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let alias_to_member_name_map = &test_data.request.alias_to_member_name_map; + let annotation = &test_data.request.annotation; + let query = &test_data.request.query; + let query_type = &test_data.request.query_type.clone().unwrap_or_default(); + + let (members_to_alias_map, members) = get_members( + query_type, + query, + &raw_data, + alias_to_member_name_map, + annotation, + )?; + let members_map_expected: MembersMap = HashMap::from([ + ( + "ECommerceRecordsUs2021.postalCode".to_string(), + "e_commerce_records_us2021__postal_code".to_string(), + ), + ( + "ECommerceRecordsUs2021.avg_profit".to_string(), + "e_commerce_records_us2021__avg_profit".to_string(), + ), + ]); + assert_eq!(members_to_alias_map, members_map_expected); + assert_eq!(members.len(), 2); + Ok(()) + } + + #[test] + fn test_get_members_compare_date_range_empty_dataset() -> Result<()> { + let test_data = TEST_SUITE_DATA + .get(&"compare_date_range_count_by_order_date".to_string()) + .unwrap() + .clone(); + let alias_to_member_name_map = &test_data.request.alias_to_member_name_map; + let annotation = &test_data.request.annotation; + let query = &test_data.request.query; + let query_type = &test_data.request.query_type.clone().unwrap_or_default(); + + let (members_to_alias_map, members) = get_members( + query_type, + query, + &QueryResult { + columns: vec![], + rows: vec![], + columns_pos: HashMap::new(), + }, + alias_to_member_name_map, + annotation, + )?; + assert_eq!(members_to_alias_map.len(), 0); + assert_eq!(members.len(), 0); + Ok(()) + } + + #[test] + fn test_get_members_compare_date_range_filled_dataset() -> Result<()> { + let test_data = TEST_SUITE_DATA + .get(&"compare_date_range_count_by_order_date".to_string()) + .unwrap() + .clone(); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let alias_to_member_name_map = &test_data.request.alias_to_member_name_map; + let annotation = &test_data.request.annotation; + let query = &test_data.request.query; + let query_type = &test_data.request.query_type.clone().unwrap_or_default(); + + let (members_to_alias_map, members) = get_members( + query_type, + query, + &raw_data, + alias_to_member_name_map, + annotation, + )?; + let members_map_expected: MembersMap = HashMap::from([ + ( + "ECommerceRecordsUs2021.orderDate.day".to_string(), + "e_commerce_records_us2021__order_date_day".to_string(), + ), + ( + "ECommerceRecordsUs2021.orderDate".to_string(), + "e_commerce_records_us2021__order_date_day".to_string(), + ), + ( + "ECommerceRecordsUs2021.count".to_string(), + "e_commerce_records_us2021__count".to_string(), + ), + ( + "compareDateRange".to_string(), + "compareDateRangeQuery".to_string(), + ), + ]); + assert_eq!(members_to_alias_map, members_map_expected); + assert_eq!(members.len(), 4); + Ok(()) + } + + #[test] + fn test_get_members_blending_query_empty_dataset() -> Result<()> { + let test_data = TEST_SUITE_DATA + .get( + &"blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode" + .to_string(), + ) + .unwrap() + .clone(); + let alias_to_member_name_map = &test_data.request.alias_to_member_name_map; + let annotation = &test_data.request.annotation; + let query = &test_data.request.query; + let query_type = &test_data.request.query_type.clone().unwrap_or_default(); + + let (members_to_alias_map, members) = get_members( + query_type, + query, + &QueryResult { + columns: vec![], + rows: vec![], + columns_pos: HashMap::new(), + }, + alias_to_member_name_map, + annotation, + )?; + assert_eq!(members_to_alias_map.len(), 0); + assert_eq!(members.len(), 0); + Ok(()) + } + + #[test] + fn test_get_members_blending_query_filled_dataset() -> Result<()> { + let test_data = TEST_SUITE_DATA + .get( + &"blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode" + .to_string(), + ) + .unwrap() + .clone(); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let alias_to_member_name_map = &test_data.request.alias_to_member_name_map; + let annotation = &test_data.request.annotation; + let query = &test_data.request.query; + let query_type = &test_data.request.query_type.clone().unwrap_or_default(); + + let (members_to_alias_map, members) = get_members( + query_type, + query, + &raw_data, + alias_to_member_name_map, + annotation, + )?; + let members_map_expected: HashMap = HashMap::from([ + ( + "ECommerceRecordsUs2021.orderDate.month".to_string(), + "e_commerce_records_us2021__order_date_month".to_string(), + ), + ( + "ECommerceRecordsUs2021.orderDate".to_string(), + "e_commerce_records_us2021__order_date_month".to_string(), + ), + ( + "ECommerceRecordsUs2021.avg_discount".to_string(), + "e_commerce_records_us2021__avg_discount".to_string(), + ), + ( + "time.month".to_string(), + "e_commerce_records_us2021__order_date_month".to_string(), + ), + ]); + assert_eq!(members_to_alias_map, members_map_expected); + assert_eq!(members.len(), 4); + Ok(()) + } + + #[test] + fn test_get_compact_row_regular_profit_by_postal_code() -> Result<()> { + let test_data = TEST_SUITE_DATA + .get(&"regular_profit_by_postal_code".to_string()) + .unwrap() + .clone(); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let alias_to_member_name_map = &test_data.request.alias_to_member_name_map; + let annotation = &test_data.request.annotation; + let query = test_data.request.query.clone(); + let query_type = &test_data.request.query_type.clone().unwrap_or_default(); + let time_dimensions = &test_data.request.query.time_dimensions.unwrap(); + + let (members_to_alias_map, members) = get_members( + query_type, + &query, + &raw_data, + alias_to_member_name_map, + annotation, + )?; + let res = get_compact_row( + &members_to_alias_map, + &annotation, + &query_type, + &members, + Some(time_dimensions), + &raw_data.rows[0], + &raw_data.columns_pos, + )?; + + let members_map_expected = HashMap::from([ + ( + "ECommerceRecordsUs2021.postalCode".to_string(), + DBResponsePrimitive::String("95823".to_string()), + ), + ( + "ECommerceRecordsUs2021.avg_profit".to_string(), + DBResponsePrimitive::String("646.1258666666666667".to_string()), + ), + ]); + + assert_eq!(res.len(), members_map_expected.len()); + for (i, val) in members.iter().enumerate() { + assert_eq!(res[i], members_map_expected.get(val).unwrap().clone()); + } + + Ok(()) + } + + #[test] + fn test_get_compact_row_regular_discount_by_city() -> Result<()> { + let test_data = TEST_SUITE_DATA + .get(&"regular_discount_by_city".to_string()) + .unwrap() + .clone(); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let alias_to_member_name_map = &test_data.request.alias_to_member_name_map; + let annotation = &test_data.request.annotation; + let query = test_data.request.query.clone(); + let query_type = &test_data.request.query_type.clone().unwrap_or_default(); + let time_dimensions = &test_data.request.query.time_dimensions.unwrap(); + + let (members_to_alias_map, members) = get_members( + query_type, + &query, + &raw_data, + alias_to_member_name_map, + annotation, + )?; + let res = get_compact_row( + &members_to_alias_map, + &annotation, + &query_type, + &members, + Some(time_dimensions), + &raw_data.rows[0], + &raw_data.columns_pos, + )?; + + let members_map_expected = HashMap::from([ + ( + "ECommerceRecordsUs2021.city".to_string(), + DBResponsePrimitive::String("Missouri City".to_string()), + ), + ( + "ECommerceRecordsUs2021.avg_discount".to_string(), + DBResponsePrimitive::String("0.80000000000000000000".to_string()), + ), + ]); + + assert_eq!(res.len(), members_map_expected.len()); + for (i, val) in members.iter().enumerate() { + assert_eq!(res[i], members_map_expected.get(val).unwrap().clone()); + } + + Ok(()) + } + + #[test] + fn test_get_compact_row_compare_date_range_count_by_order_date() -> Result<()> { + let test_data = TEST_SUITE_DATA + .get(&"compare_date_range_count_by_order_date".to_string()) + .unwrap() + .clone(); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let alias_to_member_name_map = &test_data.request.alias_to_member_name_map; + let annotation = &test_data.request.annotation; + let query = test_data.request.query.clone(); + let query_type = &test_data.request.query_type.clone().unwrap_or_default(); + let time_dimensions = &test_data.request.query.time_dimensions.unwrap(); + + let (members_to_alias_map, members) = get_members( + query_type, + &query, + &raw_data, + alias_to_member_name_map, + annotation, + )?; + let res = get_compact_row( + &members_to_alias_map, + &annotation, + &query_type, + &members, + Some(time_dimensions), + &raw_data.rows[0], + &raw_data.columns_pos, + )?; + + let members_map_expected = HashMap::from([ + ( + "ECommerceRecordsUs2021.orderDate.day".to_string(), + DBResponsePrimitive::String("2020-01-01T00:00:00.000".to_string()), + ), + ( + "ECommerceRecordsUs2021.orderDate".to_string(), + DBResponsePrimitive::String("2020-01-01T00:00:00.000".to_string()), + ), + ( + "ECommerceRecordsUs2021.count".to_string(), + DBResponsePrimitive::String("10".to_string()), + ), + ( + "compareDateRange".to_string(), + DBResponsePrimitive::String( + "2020-01-01T00:00:00.000 - 2020-01-31T23:59:59.999".to_string(), + ), + ), + ]); + + assert_eq!(res.len(), members_map_expected.len()); + for (i, val) in members.iter().enumerate() { + assert_eq!(res[i], members_map_expected.get(val).unwrap().clone()); + } + + let res = get_compact_row( + &members_to_alias_map, + &annotation, + &query_type, + &members, + Some(time_dimensions), + &raw_data.rows[1], + &raw_data.columns_pos, + )?; + + let members_map_expected = HashMap::from([ + ( + "ECommerceRecordsUs2021.orderDate.day".to_string(), + DBResponsePrimitive::String("2020-01-02T00:00:00.000".to_string()), + ), + ( + "ECommerceRecordsUs2021.orderDate".to_string(), + DBResponsePrimitive::String("2020-01-02T00:00:00.000".to_string()), + ), + ( + "ECommerceRecordsUs2021.count".to_string(), + DBResponsePrimitive::String("8".to_string()), + ), + ( + "compareDateRange".to_string(), + DBResponsePrimitive::String( + "2020-01-01T00:00:00.000 - 2020-01-31T23:59:59.999".to_string(), + ), + ), + ]); + + assert_eq!(res.len(), members_map_expected.len()); + for (i, val) in members.iter().enumerate() { + assert_eq!(res[i], members_map_expected.get(val).unwrap().clone()); + } + + Ok(()) + } + + #[test] + fn test_get_compact_row_blending_query_avg_discount() -> Result<()> { + let test_data = TEST_SUITE_DATA + .get( + &"blending_query_avg_discount_by_date_range_for_the_first_and_standard_ship_mode" + .to_string(), + ) + .unwrap() + .clone(); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let alias_to_member_name_map = &test_data.request.alias_to_member_name_map; + let annotation = &test_data.request.annotation; + let query = test_data.request.query.clone(); + let query_type = &test_data.request.query_type.clone().unwrap_or_default(); + let time_dimensions = &test_data.request.query.time_dimensions.unwrap(); + + let (members_to_alias_map, members) = get_members( + query_type, + &query, + &raw_data, + alias_to_member_name_map, + annotation, + )?; + let res = get_compact_row( + &members_to_alias_map, + &annotation, + &query_type, + &members, + Some(time_dimensions), + &raw_data.rows[0], + &raw_data.columns_pos, + )?; + + let members_map_expected = HashMap::from([ + ( + "ECommerceRecordsUs2021.orderDate.month".to_string(), + DBResponsePrimitive::String("2020-01-01T00:00:00.000".to_string()), + ), + ( + "ECommerceRecordsUs2021.orderDate".to_string(), + DBResponsePrimitive::String("2020-01-01T00:00:00.000".to_string()), + ), + ( + "ECommerceRecordsUs2021.avg_discount".to_string(), + DBResponsePrimitive::String("0.15638297872340425532".to_string()), + ), + ( + "time.month".to_string(), + DBResponsePrimitive::String("2020-01-01T00:00:00.000".to_string()), + ), + ]); + + assert_eq!(res.len(), members_map_expected.len()); + for (i, val) in members.iter().enumerate() { + assert_eq!(res[i], members_map_expected.get(val).unwrap().clone()); + } + Ok(()) + } + + #[test] + fn test_get_vanilla_row_regular_discount_by_city() -> Result<()> { + let test_data = TEST_SUITE_DATA + .get(&"regular_discount_by_city".to_string()) + .unwrap() + .clone(); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let alias_to_member_name_map = &test_data.request.alias_to_member_name_map; + let annotation = &test_data.request.annotation; + let query = test_data.request.query.clone(); + let query_type = &test_data.request.query_type.clone().unwrap_or_default(); + + let res = get_vanilla_row( + &alias_to_member_name_map, + &annotation, + &query_type, + &query, + &raw_data.rows[0], + &raw_data.columns_pos, + )?; + let expected = HashMap::from([ + ( + "ECommerceRecordsUs2021.city".to_string(), + DBResponsePrimitive::String("Missouri City".to_string()), + ), + ( + "ECommerceRecordsUs2021.avg_discount".to_string(), + DBResponsePrimitive::String("0.80000000000000000000".to_string()), + ), + ]); + assert_eq!(res, expected); + Ok(()) + } + + #[test] + fn test_get_vanilla_row_regular_discount_by_city_to_fail_member() -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get(&"regular_discount_by_city".to_string()) + .unwrap() + .clone(); + test_data + .request + .alias_to_member_name_map + .remove(&"e_commerce_records_us2021__avg_discount".to_string()); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let alias_to_member_name_map = &test_data.request.alias_to_member_name_map; + let annotation = &test_data.request.annotation; + let query = test_data.request.query.clone(); + let query_type = &test_data.request.query_type.clone().unwrap_or_default(); + + match get_vanilla_row( + &alias_to_member_name_map, + &annotation, + &query_type, + &query, + &raw_data.rows[0], + &raw_data.columns_pos, + ) { + Ok(_) => Err(TestError("get_vanilla_row() should fail ".to_string()).into()), + Err(err) => { + assert!(err.to_string().contains("Missing member name for alias")); + Ok(()) + } + } + } + + #[test] + fn test_get_vanilla_row_regular_discount_by_city_to_fail_annotation() -> Result<()> { + let mut test_data = TEST_SUITE_DATA + .get(&"regular_discount_by_city".to_string()) + .unwrap() + .clone(); + test_data + .request + .annotation + .remove(&"ECommerceRecordsUs2021.avg_discount".to_string()); + let raw_data = QueryResult::from_js_raw_data(test_data.query_result.clone())?; + let alias_to_member_name_map = &test_data.request.alias_to_member_name_map; + let annotation = &test_data.request.annotation; + let query = test_data.request.query.clone(); + let query_type = &test_data.request.query_type.clone().unwrap_or_default(); + + match get_vanilla_row( + &alias_to_member_name_map, + &annotation, + &query_type, + &query, + &raw_data.rows[0], + &raw_data.columns_pos, + ) { + Ok(_) => Err(TestError("get_vanilla_row() should fail ".to_string()).into()), + Err(err) => { + assert!(err.to_string().contains("You requested hidden member")); + Ok(()) + } + } + } +} diff --git a/rust/cubeorchestrator/src/transport.rs b/rust/cubeorchestrator/src/transport.rs new file mode 100644 index 0000000000000..5bcd358a28a40 --- /dev/null +++ b/rust/cubeorchestrator/src/transport.rs @@ -0,0 +1,290 @@ +use crate::query_result_transform::DBResponsePrimitive; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::{collections::HashMap, fmt::Display}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ResultType { + Default, + Compact, +} + +#[derive(Default, Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum QueryType { + #[default] + RegularQuery, + CompareDateRangeQuery, + BlendingQuery, +} + +impl Display for QueryType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let str = serde_json::to_value(self) + .unwrap() + .as_str() + .unwrap() + .to_string(); + write!(f, "{}", str) + } +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum MemberType { + Measures, + Dimensions, + Segments, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum FilterOperator { + Equals, + NotEquals, + Contains, + NotContains, + In, + NotIn, + Gt, + Gte, + Lt, + Lte, + Set, + NotSet, + InDateRange, + NotInDateRange, + OnTheDate, + BeforeDate, + BeforeOrOnDate, + AfterDate, + AfterOrOnDate, + MeasureFilter, + EndsWith, + NotEndsWith, + StartsWith, + NotStartsWith, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct QueryFilter { + pub member: String, + pub operator: FilterOperator, + pub values: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Hash, Eq, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct GroupingSet { + pub group_type: String, + pub id: u32, + pub sub_id: Option, +} + +// We can do nothing with JS functions here, +// but to keep DTOs in sync with reality, let's keep it. +pub type JsFunction = String; + +#[derive(Debug, Clone, Serialize, Deserialize, Hash, Eq, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct MemberExpression { + // Made as Option and JsValueDeserializer set's it to None. + pub expression: Option, + pub cube_name: String, + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub expression_name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub definition: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub grouping_set: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Hash, Eq, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct ParsedMemberExpression { + pub expression: Vec, + pub cube_name: String, + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub expression_name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub definition: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub grouping_set: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct QueryTimeDimension { + pub dimension: String, + pub date_range: Option>, + pub compare_date_range: Option>, + pub granularity: Option, +} + +pub type AliasToMemberMap = HashMap; + +pub type MembersMap = HashMap; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GranularityMeta { + pub name: String, + pub title: String, + pub interval: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub offset: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub origin: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ConfigItem { + #[serde(skip_serializing_if = "Option::is_none")] + pub title: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub short_title: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + #[serde(rename = "type")] + #[serde(skip_serializing_if = "Option::is_none")] + pub member_type: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub format: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub meta: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub drill_members: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub drill_members_grouped: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub granularities: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DrillMembersGrouped { + #[serde(skip_serializing_if = "Option::is_none")] + pub measures: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub dimensions: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AnnotatedConfigItem { + #[serde(skip_serializing_if = "Option::is_none")] + pub title: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub short_title: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + #[serde(rename = "type")] + #[serde(skip_serializing_if = "Option::is_none")] + pub member_type: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub format: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub meta: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub drill_members: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub drill_members_grouped: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub granularity: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Order { + pub id: String, + pub desc: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NormalizedQueryFilter { + pub member: String, + pub operator: FilterOperator, + #[serde(skip_serializing_if = "Option::is_none")] + pub values: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub dimension: Option, +} + +// TODO: Not used, as all members are made as Strings for now +// XXX: Omitted function variant +#[derive(Debug, Clone, Serialize, Deserialize, Hash, Eq, PartialEq)] +#[serde(untagged)] +pub enum MemberOrMemberExpression { + Member(String), + ParsedMemberExpression(ParsedMemberExpression), + MemberExpression(MemberExpression), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LogicalAndFilter { + pub and: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LogicalOrFilter { + pub or: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum QueryFilterOrLogicalFilter { + QueryFilter(QueryFilter), + LogicalAndFilter(LogicalAndFilter), + LogicalOrFilter(LogicalOrFilter), + NormalizedQueryFilter(NormalizedQueryFilter), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NormalizedQuery { + #[serde(skip_serializing_if = "Option::is_none")] + pub measures: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub dimensions: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub time_dimensions: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub segments: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub limit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub offset: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub total: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub total_query: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub timezone: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub renew_query: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ungrouped: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub response_format: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub filters: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub row_limit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub order: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub query_type: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TransformDataRequest { + pub alias_to_member_name_map: HashMap, + pub annotation: HashMap, + pub query: NormalizedQuery, + pub query_type: Option, + pub res_type: Option, +} + +pub type JsRawData = Vec>; diff --git a/rust/cubeshared/.gitignore b/rust/cubeshared/.gitignore new file mode 100644 index 0000000000000..2a0a960cb1ec5 --- /dev/null +++ b/rust/cubeshared/.gitignore @@ -0,0 +1,3 @@ +/target +/.idea +.vscode diff --git a/rust/cubeshared/CHANGELOG.md b/rust/cubeshared/CHANGELOG.md new file mode 100644 index 0000000000000..f05ecfa971fe9 --- /dev/null +++ b/rust/cubeshared/CHANGELOG.md @@ -0,0 +1,2 @@ +# ChangeLog + diff --git a/rust/cubeshared/Cargo.lock b/rust/cubeshared/Cargo.lock new file mode 100644 index 0000000000000..e7d949fe75ccd --- /dev/null +++ b/rust/cubeshared/Cargo.lock @@ -0,0 +1,41 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "cubeshared" +version = "0.1.0" +dependencies = [ + "flatbuffers", +] + +[[package]] +name = "flatbuffers" +version = "23.5.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640" +dependencies = [ + "bitflags", + "rustc_version", +] + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "semver" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" diff --git a/rust/cubeshared/Cargo.toml b/rust/cubeshared/Cargo.toml new file mode 100644 index 0000000000000..b40a0b5739708 --- /dev/null +++ b/rust/cubeshared/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "cubeshared" +version = "0.1.0" +edition = "2021" + +[dependencies] +flatbuffers = "23.1.21" diff --git a/rust/cubeshared/flatbuffers-codegen.sh b/rust/cubeshared/flatbuffers-codegen.sh new file mode 100755 index 0000000000000..e0a62d0d507a5 --- /dev/null +++ b/rust/cubeshared/flatbuffers-codegen.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +cd ./src/codegen || exit 1 +flatc --rust http_message.fbs diff --git a/rust/cubeshared/rust-toolchain.toml b/rust/cubeshared/rust-toolchain.toml new file mode 100644 index 0000000000000..040357e9b1d43 --- /dev/null +++ b/rust/cubeshared/rust-toolchain.toml @@ -0,0 +1,5 @@ +[toolchain] +#channel = "stable" +channel = "nightly-2024-07-15" +components = ["rustfmt", "clippy"] +profile = "minimal" diff --git a/rust/cubeshared/rustfmt.toml b/rust/cubeshared/rustfmt.toml new file mode 100644 index 0000000000000..d9ba5fdb90ba3 --- /dev/null +++ b/rust/cubeshared/rustfmt.toml @@ -0,0 +1 @@ +imports_granularity = "Crate" \ No newline at end of file diff --git a/rust/cubestore/cubestore/src/codegen/http_message.fbs b/rust/cubeshared/src/codegen/http_message.fbs similarity index 100% rename from rust/cubestore/cubestore/src/codegen/http_message.fbs rename to rust/cubeshared/src/codegen/http_message.fbs diff --git a/rust/cubestore/cubestore/src/codegen/http_message_generated.rs b/rust/cubeshared/src/codegen/http_message_generated.rs similarity index 100% rename from rust/cubestore/cubestore/src/codegen/http_message_generated.rs rename to rust/cubeshared/src/codegen/http_message_generated.rs diff --git a/rust/cubestore/cubestore/src/codegen/mod.rs b/rust/cubeshared/src/codegen/mod.rs similarity index 71% rename from rust/cubestore/cubestore/src/codegen/mod.rs rename to rust/cubeshared/src/codegen/mod.rs index 1b691689fb9aa..fa722ae71b1e7 100644 --- a/rust/cubestore/cubestore/src/codegen/mod.rs +++ b/rust/cubeshared/src/codegen/mod.rs @@ -1,3 +1,4 @@ +#[allow(unused_imports)] mod http_message_generated; pub use http_message_generated::*; diff --git a/rust/cubeshared/src/lib.rs b/rust/cubeshared/src/lib.rs new file mode 100644 index 0000000000000..24ccbddd82ed0 --- /dev/null +++ b/rust/cubeshared/src/lib.rs @@ -0,0 +1 @@ +pub mod codegen; diff --git a/rust/cubesql/Cargo.lock b/rust/cubesql/Cargo.lock index 1cb06b84594cd..51c26fde43a59 100644 --- a/rust/cubesql/Cargo.lock +++ b/rust/cubesql/Cargo.lock @@ -188,7 +188,7 @@ checksum = "531b97fb4cd3dfdce92c35dedbfdc1f0b9d8091c8ca943d6dae340ef5012d514" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.87", ] [[package]] @@ -829,7 +829,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.76", + "syn 2.0.87", ] [[package]] @@ -846,7 +846,7 @@ checksum = "a26acccf6f445af85ea056362561a24ef56cdc15fcc685f03aec50b9c702cb6d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.87", ] [[package]] @@ -1161,7 +1161,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.87", ] [[package]] @@ -2221,7 +2221,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.87", ] [[package]] @@ -2712,29 +2712,29 @@ checksum = "58bf37232d3bb9a2c4e641ca2a11d83b5062066f88df7fed36c28772046d65ba" [[package]] name = "serde" -version = "1.0.209" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.209" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.87", ] [[package]] name = "serde_json" -version = "1.0.127" +version = "1.0.135" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad" +checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" dependencies = [ "indexmap 2.4.0", "itoa 1.0.10", @@ -2932,7 +2932,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.76", + "syn 2.0.87", ] [[package]] @@ -2971,9 +2971,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.76" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578e081a14e0cefc3279b0472138c513f37b41a08d5a3cca9b6e4e8ceb6cd525" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -3064,7 +3064,7 @@ checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.87", ] [[package]] @@ -3151,7 +3151,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.87", ] [[package]] @@ -3260,7 +3260,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.87", ] [[package]] @@ -3842,7 +3842,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.87", ] [[package]] diff --git a/rust/cubesql/cubeclient/src/models/v1_load_result.rs b/rust/cubesql/cubeclient/src/models/v1_load_result.rs index be2a78fbee94b..7b50ab1633e59 100644 --- a/rust/cubesql/cubeclient/src/models/v1_load_result.rs +++ b/rust/cubesql/cubeclient/src/models/v1_load_result.rs @@ -7,7 +7,6 @@ * * Generated by: https://openapi-generator.tech */ - #[derive(Clone, Debug, PartialEq, Default, Serialize, Deserialize)] pub struct V1LoadResult { #[serde(rename = "dataSource", skip_serializing_if = "Option::is_none")] diff --git a/rust/cubesql/cubesql/src/compile/engine/df/scan.rs b/rust/cubesql/cubesql/src/compile/engine/df/scan.rs index 33577e2e6857c..afd410898b25c 100644 --- a/rust/cubesql/cubesql/src/compile/engine/df/scan.rs +++ b/rust/cubesql/cubesql/src/compile/engine/df/scan.rs @@ -1,5 +1,5 @@ use async_trait::async_trait; -use cubeclient::models::{V1LoadRequestQuery, V1LoadResult, V1LoadResultAnnotation}; +use cubeclient::models::{V1LoadRequestQuery, V1LoadResponse}; pub use datafusion::{ arrow::{ array::{ @@ -52,7 +52,7 @@ use datafusion::{ logical_plan::JoinType, scalar::ScalarValue, }; -use serde_json::{json, Value}; +use serde_json::Value; #[derive(Debug, Clone, Eq, PartialEq)] pub enum MemberField { @@ -655,27 +655,22 @@ impl ExecutionPlan for CubeScanExecutionPlan { ))); } - let mut response = JsonValueObject::new( - load_data( - self.span_id.clone(), - request, - self.auth_context.clone(), - self.transport.clone(), - meta.clone(), - self.options.clone(), - self.wrapped_sql.clone(), - ) - .await? - .data, - ); - one_shot_stream.data = Some( - transform_response( - &mut response, - one_shot_stream.schema.clone(), - &one_shot_stream.member_fields, - ) - .map_err(|e| DataFusionError::Execution(e.message.to_string()))?, - ); + let response = load_data( + self.span_id.clone(), + request, + self.auth_context.clone(), + self.transport.clone(), + meta.clone(), + self.schema.clone(), + self.member_fields.clone(), + self.options.clone(), + self.wrapped_sql.clone(), + ) + .await?; + + // For now execute method executes only one query at a time, so we + // take the first result + one_shot_stream.data = Some(response.first().unwrap().clone()); Ok(Box::pin(CubeScanStreamRouter::new( None, @@ -840,9 +835,11 @@ async fn load_data( auth_context: AuthContextRef, transport: Arc, meta: LoadRequestMeta, + schema: SchemaRef, + member_fields: Vec, options: CubeScanOptions, sql_query: Option, -) -> ArrowResult { +) -> ArrowResult> { let no_members_query = request.measures.as_ref().map(|v| v.len()).unwrap_or(0) == 0 && request.dimensions.as_ref().map(|v| v.len()).unwrap_or(0) == 0 && request @@ -860,22 +857,27 @@ async fn load_data( data.push(serde_json::Value::Null) } - V1LoadResult::new( - V1LoadResultAnnotation { - measures: json!(Vec::::new()), - dimensions: json!(Vec::::new()), - segments: json!(Vec::::new()), - time_dimensions: json!(Vec::::new()), - }, - data, - ) + let mut response = JsonValueObject::new(data); + let rec = transform_response(&mut response, schema.clone(), &member_fields) + .map_err(|e| DataFusionError::Execution(e.message.to_string()))?; + + rec } else { let result = transport - .load(span_id, request, sql_query, auth_context, meta) - .await; - let mut response = result.map_err(|err| ArrowError::ComputeError(err.to_string()))?; - if let Some(data) = response.results.pop() { - match (options.max_records, data.data.len()) { + .load( + span_id, + request, + sql_query, + auth_context, + meta, + schema, + member_fields, + ) + .await + .map_err(|err| ArrowError::ComputeError(err.to_string()))?; + let response = result.first(); + if let Some(data) = response.cloned() { + match (options.max_records, data.num_rows()) { (Some(max_records), len) if len >= max_records => { return Err(ArrowError::ComputeError(format!("One of the Cube queries exceeded the maximum row limit ({}). JOIN/UNION is not possible as it will produce incorrect results. Try filtering the results more precisely or moving post-processing functions to an outer query.", max_records))); } @@ -890,7 +892,7 @@ async fn load_data( } }; - Ok(result) + Ok(vec![result]) } fn load_to_stream_sync(one_shot_stream: &mut CubeScanOneShotStream) -> Result<()> { @@ -899,6 +901,8 @@ fn load_to_stream_sync(one_shot_stream: &mut CubeScanOneShotStream) -> Result<() let auth = one_shot_stream.auth_context.clone(); let transport = one_shot_stream.transport.clone(); let meta = one_shot_stream.meta.clone(); + let schema = one_shot_stream.schema.clone(); + let member_fields = one_shot_stream.member_fields.clone(); let options = one_shot_stream.options.clone(); let wrapped_sql = one_shot_stream.wrapped_sql.clone(); @@ -910,22 +914,17 @@ fn load_to_stream_sync(one_shot_stream: &mut CubeScanOneShotStream) -> Result<() auth, transport, meta, + schema, + member_fields, options, wrapped_sql, )) }) .join() - .map_err(|_| DataFusionError::Execution(format!("Can't load to stream")))?; - - let mut response = JsonValueObject::new(res.unwrap().data); - one_shot_stream.data = Some( - transform_response( - &mut response, - one_shot_stream.schema.clone(), - &one_shot_stream.member_fields, - ) - .map_err(|e| DataFusionError::Execution(e.message.to_string()))?, - ); + .map_err(|_| DataFusionError::Execution(format!("Can't load to stream")))??; + + let response = res.first(); + one_shot_stream.data = Some(response.cloned().unwrap()); Ok(()) } @@ -1128,7 +1127,7 @@ pub fn transform_response( )) })?; // TODO switch parsing to microseconds - if timestamp.timestamp_millis() > (((1i64) << 62) / 1_000_000) { + if timestamp.and_utc().timestamp_millis() > (((1i64) << 62) / 1_000_000) { builder.append_null()?; } else if let Some(nanos) = timestamp.timestamp_nanos_opt() { builder.append_value(nanos)?; @@ -1170,10 +1169,10 @@ pub fn transform_response( )) })?; // TODO switch parsing to microseconds - if timestamp.timestamp_millis() > (((1 as i64) << 62) / 1_000_000) { + if timestamp.and_utc().timestamp_millis() > (((1 as i64) << 62) / 1_000_000) { builder.append_null()?; } else { - builder.append_value(timestamp.timestamp_millis())?; + builder.append_value(timestamp.and_utc().timestamp_millis())?; } }, }, @@ -1331,6 +1330,21 @@ pub fn transform_response( Ok(RecordBatch::try_new(schema.clone(), columns)?) } +pub fn convert_transport_response( + response: V1LoadResponse, + schema: SchemaRef, + member_fields: Vec, +) -> std::result::Result, CubeError> { + response + .results + .into_iter() + .map(|r| { + let mut response = JsonValueObject::new(r.data.clone()); + transform_response(&mut response, schema.clone(), &member_fields) + }) + .collect::, CubeError>>() +} + #[cfg(test)] mod tests { use super::*; @@ -1394,9 +1408,12 @@ mod tests { _sql_query: Option, _ctx: AuthContextRef, _meta_fields: LoadRequestMeta, - ) -> Result { + schema: SchemaRef, + member_fields: Vec, + ) -> Result, CubeError> { let response = r#" - { + { + "results": [{ "annotation": { "measures": [], "dimensions": [], @@ -1410,17 +1427,13 @@ mod tests { {"KibanaSampleDataEcommerce.count": null, "KibanaSampleDataEcommerce.maxPrice": null, "KibanaSampleDataEcommerce.isBool": "true", "KibanaSampleDataEcommerce.orderDate": "9999-12-31 00:00:00.000", "KibanaSampleDataEcommerce.city": "City 4"}, {"KibanaSampleDataEcommerce.count": null, "KibanaSampleDataEcommerce.maxPrice": null, "KibanaSampleDataEcommerce.isBool": "false", "KibanaSampleDataEcommerce.orderDate": null, "KibanaSampleDataEcommerce.city": null} ] - } + }] + } "#; - let result: V1LoadResult = serde_json::from_str(response).unwrap(); - - Ok(V1LoadResponse { - pivot_query: None, - slow_query: None, - query_type: None, - results: vec![result], - }) + let result: V1LoadResponse = serde_json::from_str(response).unwrap(); + convert_transport_response(result, schema.clone(), member_fields) + .map_err(|err| CubeError::user(err.to_string())) } async fn load_stream( diff --git a/rust/cubesql/cubesql/src/compile/engine/udf/common.rs b/rust/cubesql/cubesql/src/compile/engine/udf/common.rs index 127fed402cc73..20f57f81379bb 100644 --- a/rust/cubesql/cubesql/src/compile/engine/udf/common.rs +++ b/rust/cubesql/cubesql/src/compile/engine/udf/common.rs @@ -1029,6 +1029,7 @@ pub fn create_date_udf() -> ScalarUDF { builder.append_value( NaiveDateTime::parse_from_str(strings.value(i), "%Y-%m-%d %H:%M:%S%.f") .map_err(|e| DataFusionError::Execution(e.to_string()))? + .and_utc() .timestamp_nanos_opt() .unwrap(), )?; @@ -1233,6 +1234,7 @@ macro_rules! date_math_udf { let interval = intervals.value(i).into(); builder.append_value( $FUN(timestamp, interval, $IS_ADD)? + .and_utc() .timestamp_nanos_opt() .unwrap(), )?; @@ -1569,7 +1571,7 @@ pub fn create_str_to_date_udf() -> ScalarUDF { })?; Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( - Some(res.timestamp_nanos_opt().unwrap()), + Some(res.and_utc().timestamp_nanos_opt().unwrap()), None, ))) }); @@ -2339,7 +2341,7 @@ macro_rules! generate_series_helper_timestamp { )) })?; let res = date_addsub_month_day_nano(current_dt, $STEP, true)?; - $CURRENT = res.timestamp_nanos_opt().unwrap() as $PRIMITIVE_TYPE; + $CURRENT = res.and_utc().timestamp_nanos_opt().unwrap() as $PRIMITIVE_TYPE; }; } @@ -3433,6 +3435,7 @@ pub fn create_date_to_timestamp_udf() -> ScalarUDF { )?; Ok(Some( NaiveDateTime::new(date, time) + .and_utc() .timestamp_nanos_opt() .unwrap(), )) diff --git a/rust/cubesql/cubesql/src/compile/rewrite/rules/filters.rs b/rust/cubesql/cubesql/src/compile/rewrite/rules/filters.rs index f859ef2e31c9f..bac53c2026a93 100644 --- a/rust/cubesql/cubesql/src/compile/rewrite/rules/filters.rs +++ b/rust/cubesql/cubesql/src/compile/rewrite/rules/filters.rs @@ -4118,8 +4118,8 @@ impl FilterRules { }; let (Some(start_date), Some(end_date)) = ( - start_date.timestamp_nanos_opt(), - end_date.timestamp_nanos_opt(), + start_date.and_utc().timestamp_nanos_opt(), + end_date.and_utc().timestamp_nanos_opt(), ) else { return false; }; diff --git a/rust/cubesql/cubesql/src/compile/test/mod.rs b/rust/cubesql/cubesql/src/compile/test/mod.rs index cc62e88d7b897..8c07ac6b7041a 100644 --- a/rust/cubesql/cubesql/src/compile/test/mod.rs +++ b/rust/cubesql/cubesql/src/compile/test/mod.rs @@ -44,6 +44,9 @@ pub mod test_user_change; #[cfg(test)] pub mod test_wrapper; pub mod utils; +use crate::compile::{ + arrow::record_batch::RecordBatch, engine::df::scan::convert_transport_response, +}; pub use utils::*; pub fn get_test_meta() -> Vec { @@ -804,7 +807,9 @@ impl TransportService for TestConnectionTransport { sql_query: Option, ctx: AuthContextRef, meta: LoadRequestMeta, - ) -> Result { + schema: SchemaRef, + member_fields: Vec, + ) -> Result, CubeError> { { let mut calls = self.load_calls.lock().await; calls.push(TestTransportLoadCall { @@ -822,12 +827,19 @@ impl TransportService for TestConnectionTransport { } let mocks = self.load_mocks.lock().await; - let Some((_req, res)) = mocks.iter().find(|(req, _res)| req == &query) else { + let Some(res) = mocks + .iter() + .find(|(req, _res)| req == &query) + .map(|(_req, res)| { + convert_transport_response(res.clone(), schema.clone(), member_fields) + }) + else { return Err(CubeError::internal(format!( "Unexpected query in test transport: {query:?}" ))); }; - Ok(res.clone()) + + res } async fn load_stream( diff --git a/rust/cubesql/cubesql/src/transport/service.rs b/rust/cubesql/cubesql/src/transport/service.rs index 7812eefebe839..85d9d270910f9 100644 --- a/rust/cubesql/cubesql/src/transport/service.rs +++ b/rust/cubesql/cubesql/src/transport/service.rs @@ -28,15 +28,13 @@ use uuid::Uuid; use crate::{ compile::{ engine::df::{ - scan::MemberField, + scan::{convert_transport_response, MemberField}, wrapper::{GroupingSetDesc, GroupingSetType, SqlQuery}, }, rewrite::LikeType, }, sql::{AuthContextRef, HttpAuthContext}, - transport::{ - MetaContext, TransportLoadRequest, TransportLoadRequestQuery, TransportLoadResponse, - }, + transport::{MetaContext, TransportLoadRequest, TransportLoadRequestQuery}, CubeError, RWLockAsync, }; @@ -142,7 +140,9 @@ pub trait TransportService: Send + Sync + Debug { sql_query: Option, ctx: AuthContextRef, meta_fields: LoadRequestMeta, - ) -> Result; + schema: SchemaRef, + member_fields: Vec, + ) -> Result, CubeError>; async fn load_stream( &self, @@ -280,7 +280,9 @@ impl TransportService for HttpTransport { _sql_query: Option, ctx: AuthContextRef, meta: LoadRequestMeta, - ) -> Result { + schema: SchemaRef, + member_fields: Vec, + ) -> Result, CubeError> { if meta.change_user().is_some() { return Err(CubeError::internal( "Changing security context (__user) is not supported in the standalone mode" @@ -296,7 +298,7 @@ impl TransportService for HttpTransport { let response = cube_api::load_v1(&self.get_client_config_for_ctx(ctx), Some(request)).await?; - Ok(response) + convert_transport_response(response, schema, member_fields) } async fn load_stream( diff --git a/rust/cubesqlplanner/Cargo.lock b/rust/cubesqlplanner/Cargo.lock index 55b1401878539..c2732a97cef95 100644 --- a/rust/cubesqlplanner/Cargo.lock +++ b/rust/cubesqlplanner/Cargo.lock @@ -175,7 +175,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", ] [[package]] @@ -186,7 +186,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", ] [[package]] @@ -336,7 +336,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", "syn_derive", ] @@ -1102,7 +1102,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", ] [[package]] @@ -1793,7 +1793,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6813fde79b646e47e7ad75f480aa80ef76a5d9599e2717407961531169ee38b" dependencies = [ "quote", - "syn 2.0.69", + "syn 2.0.95", "syn-mid", ] @@ -2030,7 +2030,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", ] [[package]] @@ -2113,7 +2113,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", ] [[package]] @@ -2232,9 +2232,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] @@ -2647,32 +2647,33 @@ checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" [[package]] name = "serde" -version = "1.0.209" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.209" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", ] [[package]] name = "serde_json" -version = "1.0.120" +version = "1.0.135" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" +checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" dependencies = [ "indexmap 2.2.6", "itoa", + "memchr", "ryu", "serde", ] @@ -2837,7 +2838,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.69", + "syn 2.0.95", ] [[package]] @@ -2876,9 +2877,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.69" +version = "2.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201fcda3845c23e8212cd466bfebf0bd20694490fc0356ae8e428e0824a915a6" +checksum = "46f71c0377baf4ef1cc3e3402ded576dccc315800fbc62dfc7fe04b009773b4a" dependencies = [ "proc-macro2", "quote", @@ -2893,7 +2894,7 @@ checksum = "b5dc35bb08dd1ca3dfb09dce91fd2d13294d6711c88897d9a9d60acf39bce049" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", ] [[package]] @@ -2905,7 +2906,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", ] [[package]] @@ -2971,7 +2972,7 @@ checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", ] [[package]] @@ -3039,7 +3040,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", ] [[package]] @@ -3141,7 +3142,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", ] [[package]] @@ -3375,7 +3376,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", "wasm-bindgen-shared", ] @@ -3409,7 +3410,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3663,7 +3664,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.95", ] [[package]] diff --git a/rust/cubestore/Cargo.lock b/rust/cubestore/Cargo.lock index 89352f63787dc..c6c32d50505b0 100644 --- a/rust/cubestore/Cargo.lock +++ b/rust/cubestore/Cargo.lock @@ -1148,6 +1148,13 @@ dependencies = [ "syn 1.0.107", ] +[[package]] +name = "cubeshared" +version = "0.1.0" +dependencies = [ + "flatbuffers 23.1.21", +] + [[package]] name = "cubestore" version = "0.1.0" @@ -1173,6 +1180,7 @@ dependencies = [ "cubehll", "cuberockstore", "cuberpc", + "cubeshared", "cubezetasketch", "datafusion", "deadqueue", diff --git a/rust/cubestore/Dockerfile b/rust/cubestore/Dockerfile index f522c4ef8e23f..db2ec1ec76cd1 100644 --- a/rust/cubestore/Dockerfile +++ b/rust/cubestore/Dockerfile @@ -1,15 +1,18 @@ FROM cubejs/rust-builder:bookworm-llvm-18 AS builder WORKDIR /build/cubestore -COPY Cargo.toml . -COPY Cargo.lock . -COPY cuberockstore cuberockstore -COPY cubehll cubehll -COPY cubezetasketch cubezetasketch -COPY cubedatasketches cubedatasketches -COPY cuberpc cuberpc -COPY cubestore-sql-tests cubestore-sql-tests -COPY cubestore/Cargo.toml cubestore/Cargo.toml + +COPY cubeshared /build/cubeshared + +COPY cubestore/Cargo.toml . +COPY cubestore/Cargo.lock . +COPY cubestore/cuberockstore cuberockstore +COPY cubestore/cubehll cubehll +COPY cubestore/cubezetasketch cubezetasketch +COPY cubestore/cubedatasketches cubedatasketches +COPY cubestore/cuberpc cuberpc +COPY cubestore/cubestore-sql-tests cubestore-sql-tests +COPY cubestore/cubestore/Cargo.toml cubestore/Cargo.toml RUN mkdir -p cubestore/src/bin && \ echo "fn main() {print!(\"Dummy main\");} // dummy file" > cubestore/src/bin/cubestored.rs @@ -18,8 +21,8 @@ RUN [ "$WITH_AVX2" -eq "1" ] && export RUSTFLAGS="-C target-feature=+avx2"; \ cargo build --release -p cubestore # Cube Store get version from his own package -COPY package.json package.json -COPY cubestore cubestore +COPY cubestore/package.json package.json +COPY cubestore/cubestore cubestore RUN [ "$WITH_AVX2" -eq "1" ] && export RUSTFLAGS="-C target-feature=+avx2"; \ cargo build --release -p cubestore diff --git a/rust/cubestore/cubestore/Cargo.toml b/rust/cubestore/cubestore/Cargo.toml index 2e3dc5f5caf2b..3e54d4428f00c 100644 --- a/rust/cubestore/cubestore/Cargo.toml +++ b/rust/cubestore/cubestore/Cargo.toml @@ -27,6 +27,7 @@ cuberockstore = { path = "../cuberockstore" } cubehll = { path = "../cubehll" } cubezetasketch = { path = "../cubezetasketch" } cubedatasketches = { path = "../cubedatasketches" } +cubeshared = { path = "../../cubeshared" } cuberpc = { path = "../cuberpc" } datafusion = { git = "https://github.com/cube-js/arrow-datafusion", branch = "cube", features = ["default_nulls_last"] } csv = "1.1.3" diff --git a/rust/cubestore/cubestore/src/http/mod.rs b/rust/cubestore/cubestore/src/http/mod.rs index f627a1f5f067e..e03fe51d0b425 100644 --- a/rust/cubestore/cubestore/src/http/mod.rs +++ b/rust/cubestore/cubestore/src/http/mod.rs @@ -4,11 +4,6 @@ use std::sync::Arc; use warp::{Filter, Rejection, Reply}; -use crate::codegen::{ - root_as_http_message, HttpColumnValue, HttpColumnValueArgs, HttpError, HttpErrorArgs, - HttpMessageArgs, HttpQuery, HttpQueryArgs, HttpResultSet, HttpResultSetArgs, HttpRow, - HttpRowArgs, -}; use crate::metastore::{Column, ColumnType, ImportFormat}; use crate::mysql::SqlAuthService; use crate::sql::{InlineTable, InlineTables, SqlQueryContext, SqlService}; @@ -17,6 +12,11 @@ use crate::table::{Row, TableValue}; use crate::util::WorkerLoop; use crate::CubeError; use async_std::fs::File; +use cubeshared::codegen::{ + root_as_http_message, HttpColumnValue, HttpColumnValueArgs, HttpError, HttpErrorArgs, + HttpMessageArgs, HttpQuery, HttpQueryArgs, HttpResultSet, HttpResultSetArgs, HttpRow, + HttpRowArgs, +}; use datafusion::cube_ext; use flatbuffers::{FlatBufferBuilder, ForwardsUOffset, Vector, WIPOffset}; use futures::{AsyncWriteExt, SinkExt, Stream, StreamExt}; @@ -603,10 +603,10 @@ impl HttpMessage { let args = HttpMessageArgs { message_id: self.message_id, command_type: match self.command { - HttpCommand::Query { .. } => crate::codegen::HttpCommand::HttpQuery, - HttpCommand::ResultSet { .. } => crate::codegen::HttpCommand::HttpResultSet, + HttpCommand::Query { .. } => cubeshared::codegen::HttpCommand::HttpQuery, + HttpCommand::ResultSet { .. } => cubeshared::codegen::HttpCommand::HttpResultSet, HttpCommand::CloseConnection { .. } | HttpCommand::Error { .. } => { - crate::codegen::HttpCommand::HttpError + cubeshared::codegen::HttpCommand::HttpError } }, command: match &self.command { @@ -666,7 +666,7 @@ impl HttpMessage { .as_ref() .map(|c| builder.create_string(c)), }; - let message = crate::codegen::HttpMessage::create(&mut builder, &args); + let message = cubeshared::codegen::HttpMessage::create(&mut builder, &args); builder.finish(message, None); builder.finished_data().to_vec() // TODO copy } @@ -762,7 +762,7 @@ impl HttpMessage { message_id: http_message.message_id(), connection_id: http_message.connection_id().map(|s| s.to_string()), command: match http_message.command_type() { - crate::codegen::HttpCommand::HttpQuery => { + cubeshared::codegen::HttpCommand::HttpQuery => { let query = http_message.command_as_http_query().unwrap(); let mut inline_tables = Vec::new(); if let Some(query_inline_tables) = query.inline_tables() { @@ -809,7 +809,7 @@ impl HttpMessage { trace_obj: query.trace_obj().map(|q| q.to_string()), } } - crate::codegen::HttpCommand::HttpResultSet => { + cubeshared::codegen::HttpCommand::HttpResultSet => { let result_set = http_message.command_as_http_result_set().unwrap(); let mut result_rows = Vec::new(); if let Some(rows) = result_set.rows() { @@ -857,7 +857,6 @@ impl HttpMessage { #[cfg(test)] mod tests { - use crate::codegen::{HttpMessageArgs, HttpQuery, HttpQueryArgs, HttpTable, HttpTableArgs}; use crate::config::{init_test_logger, Config}; use crate::http::{HttpCommand, HttpMessage, HttpServer}; use crate::metastore::{Column, ColumnType}; @@ -867,6 +866,9 @@ mod tests { use crate::table::{Row, TableValue}; use crate::CubeError; use async_trait::async_trait; + use cubeshared::codegen::{ + HttpMessageArgs, HttpQuery, HttpQueryArgs, HttpTable, HttpTableArgs, + }; use datafusion::cube_ext; use flatbuffers::{FlatBufferBuilder, ForwardsUOffset, Vector, WIPOffset}; use futures_util::{SinkExt, StreamExt}; @@ -973,11 +975,11 @@ mod tests { ); let args = HttpMessageArgs { message_id: 1234, - command_type: crate::codegen::HttpCommand::HttpQuery, + command_type: cubeshared::codegen::HttpCommand::HttpQuery, command: Some(query_value.as_union_value()), connection_id: Some(connection_id_offset), }; - let message = crate::codegen::HttpMessage::create(&mut builder, &args); + let message = cubeshared::codegen::HttpMessage::create(&mut builder, &args); builder.finish(message, None); let bytes = builder.finished_data().to_vec(); let message = HttpMessage::read(bytes).await.unwrap(); diff --git a/rust/cubestore/cubestore/src/lib.rs b/rust/cubestore/cubestore/src/lib.rs index 89ddb44e15599..05d24b86f0a14 100644 --- a/rust/cubestore/cubestore/src/lib.rs +++ b/rust/cubestore/cubestore/src/lib.rs @@ -38,7 +38,6 @@ use tokio::time::error::Elapsed; pub mod app_metrics; pub mod cachestore; pub mod cluster; -pub mod codegen; pub mod config; pub mod http; pub mod import;