diff --git a/.taplo.toml b/.taplo.toml new file mode 100644 index 0000000..7496901 --- /dev/null +++ b/.taplo.toml @@ -0,0 +1,45 @@ +## https://taplo.tamasfe.dev/configuration/file.html + +include = ["**/Cargo.toml"] + +[formatting] +# Align consecutive entries vertically. +align_entries = false +# Append trailing commas for multi-line arrays. +array_trailing_comma = true +# Expand arrays to multiple lines that exceed the maximum column width. +array_auto_expand = true +# Collapse arrays that don't exceed the maximum column width and don't contain comments. +array_auto_collapse = false +# Omit white space padding from single-line arrays +compact_arrays = true +# Omit white space padding from the start and end of inline tables. +compact_inline_tables = false +# Maximum column width in characters, affects array expansion and collapse, this doesn't take whitespace into account. +# Note that this is not set in stone, and works on a best-effort basis. +column_width = 120 +# Indent based on tables and arrays of tables and their subtables, subtables out of order are not indented. +indent_tables = false +# The substring that is used for indentation, should be tabs or spaces (but technically can be anything). +indent_string = ' ' +# Add trailing newline at the end of the file if not present. +trailing_newline = true +# Alphabetically reorder keys that are not separated by empty lines. +reorder_keys = false +# Maximum amount of allowed consecutive blank lines. This does not affect the whitespace at the end of the document, as it is always stripped. +allowed_blank_lines = 1 +# Use CRLF for line endings. +crlf = false + +[[rule]] +keys = [ + "build-dependencies", + "dependencies", + "dev-dependencies", + "workspace.dependencies", +] +formatting = { reorder_keys = true } + +[[rule]] +keys = ["package"] +formatting = { reorder_keys = false } diff --git a/Cargo.toml b/Cargo.toml index a819b54..a4afc50 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,10 +15,12 @@ readme = "README.md" repository = "https://github.com/datafusion-contrib/datafusion-federation" [workspace.dependencies] -async-trait = "0.1.81" +arrow = "53.0.0" +arrow-flight = { version = "53.0.0", features = ["flight-sql-experimental"] } +arrow-json = "53.0.0" async-stream = "0.3.5" +async-trait = "0.1.81" +datafusion = "42.0.0" +datafusion-federation = { path = "datafusion-federation", version = "0.2.1" } +datafusion-substrait = "42.0.0" futures = "0.3.30" -datafusion = "41.0.0" -datafusion-substrait = "41.0.0" -arrow-json = "52.2.0" -datafusion-federation = "0.2.1" diff --git a/datafusion-federation/Cargo.toml b/datafusion-federation/Cargo.toml index 33cf75a..edbb3a5 100644 --- a/datafusion-federation/Cargo.toml +++ b/datafusion-federation/Cargo.toml @@ -21,17 +21,16 @@ no-default-features = true sql = [] [dependencies] -futures.workspace = true +arrow-json.workspace = true +async-stream.workspace = true async-trait.workspace = true datafusion.workspace = true -async-stream.workspace = true -arrow-json.workspace = true - +futures.workspace = true [dev-dependencies] tokio = { version = "1.39.3", features = ["full"] } -tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } tracing = "0.1.40" +tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } [[example]] name = "df-csv" diff --git a/datafusion-federation/src/sql/mod.rs b/datafusion-federation/src/sql/mod.rs index 9489173..6694ce8 100644 --- a/datafusion-federation/src/sql/mod.rs +++ b/datafusion-federation/src/sql/mod.rs @@ -419,14 +419,6 @@ fn rewrite_table_scans_in_expr( try_cast.data_type, ))) } - Expr::Sort(sort) => { - let expr = rewrite_table_scans_in_expr(*sort.expr, known_rewrites)?; - Ok(Expr::Sort(Sort::new( - Box::new(expr), - sort.asc, - sort.nulls_first, - ))) - } Expr::ScalarFunction(sf) => { let args = sf .args @@ -451,10 +443,13 @@ fn rewrite_table_scans_in_expr( .map(Box::new); let order_by = af .order_by - .map(|e| { - e.into_iter() - .map(|e| rewrite_table_scans_in_expr(e, known_rewrites)) - .collect::>>() + .map(|s| { + s.into_iter() + .map(|s| { + rewrite_table_scans_in_expr(s.expr, known_rewrites) + .map(|e| Sort::new(e, s.asc, s.nulls_first)) + }) + .collect::>>() }) .transpose()?; Ok(Expr::AggregateFunction(AggregateFunction { @@ -480,8 +475,11 @@ fn rewrite_table_scans_in_expr( let order_by = wf .order_by .into_iter() - .map(|e| rewrite_table_scans_in_expr(e, known_rewrites)) - .collect::>>()?; + .map(|s| { + rewrite_table_scans_in_expr(s.expr, known_rewrites) + .map(|e| Sort::new(e, s.asc, s.nulls_first)) + }) + .collect::>>()?; Ok(Expr::WindowFunction(WindowFunction { fun: wf.fun, args, @@ -533,13 +531,14 @@ fn rewrite_table_scans_in_expr( is.negated, ))) } - Expr::Wildcard { qualifier } => { + Expr::Wildcard { qualifier, options } => { if let Some(rewrite) = qualifier.as_ref().and_then(|q| known_rewrites.get(q)) { Ok(Expr::Wildcard { qualifier: Some(rewrite.clone()), + options, }) } else { - Ok(Expr::Wildcard { qualifier }) + Ok(Expr::Wildcard { qualifier, options }) } } Expr::GroupingSet(gs) => match gs { diff --git a/datafusion-federation/src/table_provider.rs b/datafusion-federation/src/table_provider.rs index b820b6e..ee1ab75 100644 --- a/datafusion-federation/src/table_provider.rs +++ b/datafusion-federation/src/table_provider.rs @@ -1,4 +1,4 @@ -use std::{any::Any, sync::Arc}; +use std::{any::Any, borrow::Cow, sync::Arc}; use async_trait::async_trait; use datafusion::{ @@ -70,7 +70,7 @@ impl TableProvider for FederatedTableProviderAdaptor { self.source.table_type() } - fn get_logical_plan(&self) -> Option<&LogicalPlan> { + fn get_logical_plan(&self) -> Option> { if let Some(table_provider) = &self.table_provider { return table_provider .get_logical_plan() diff --git a/datafusion-flight-sql-server/Cargo.toml b/datafusion-flight-sql-server/Cargo.toml index 8a912a1..5275db7 100644 --- a/datafusion-flight-sql-server/Cargo.toml +++ b/datafusion-flight-sql-server/Cargo.toml @@ -11,23 +11,23 @@ name = "datafusion_flight_sql_server" path = "src/lib.rs" [dependencies] +arrow.workspace = true +arrow-flight.workspace = true datafusion.workspace = true -datafusion-substrait.workspace = true datafusion-federation = { workspace = true, features = ["sql"] } +datafusion-substrait.workspace = true futures = "0.3.30" -tonic = { version = "0.11.0", features = [ +log = "0.4.22" +once_cell = "1.19.0" +prost = "0.13.1" +tonic = { version = "0.12.1", features = [ "tls", "transport", "codegen", "prost", ] } -prost = "0.12.3" -arrow = "52.0.0" -arrow-flight = { version = "52.2.0", features = ["flight-sql-experimental"] } -log = "0.4.22" -once_cell = "1.19.0" [dev-dependencies] -tokio = { version = "1.39.3", features = ["full"] } datafusion-flight-sql-table-provider = { path = "../datafusion-flight-sql-table-provider" } +tokio = { version = "1.39.3", features = ["full"] } diff --git a/datafusion-flight-sql-table-provider/Cargo.toml b/datafusion-flight-sql-table-provider/Cargo.toml index 22b15d7..daf427b 100644 --- a/datafusion-flight-sql-table-provider/Cargo.toml +++ b/datafusion-flight-sql-table-provider/Cargo.toml @@ -6,16 +6,16 @@ license.workspace = true readme.workspace = true [dependencies] +arrow.workspace = true +arrow-flight.workspace = true async-trait.workspace = true datafusion.workspace = true datafusion-federation = { workspace = true, features = ["sql"] } futures = "0.3.30" -tonic = { version = "0.11.0", features = [ +tonic = { version = "0.12.1", features = [ "tls", "transport", "codegen", "prost", ] } -arrow = "52.0.0" -arrow-flight = { version = "52.2.0", features = ["flight-sql-experimental"] }