chore: upgrade datafusion & arrow

.taplo.toml

@@ -0,0 +1,45 @@
+## https://taplo.tamasfe.dev/configuration/file.html
+
+include = ["**/Cargo.toml"]
+
+[formatting]
+# Align consecutive entries vertically.
+align_entries = false
+# Append trailing commas for multi-line arrays.
+array_trailing_comma = true
+# Expand arrays to multiple lines that exceed the maximum column width.
+array_auto_expand = true
+# Collapse arrays that don't exceed the maximum column width and don't contain comments.
+array_auto_collapse = false
+# Omit white space padding from single-line arrays
+compact_arrays = true
+# Omit white space padding from the start and end of inline tables.
+compact_inline_tables = false
+# Maximum column width in characters, affects array expansion and collapse, this doesn't take whitespace into account.
+# Note that this is not set in stone, and works on a best-effort basis.
+column_width = 120
+# Indent based on tables and arrays of tables and their subtables, subtables out of order are not indented.
+indent_tables = false
+# The substring that is used for indentation, should be tabs or spaces (but technically can be anything).
+indent_string = '    '
+# Add trailing newline at the end of the file if not present.
+trailing_newline = true
+# Alphabetically reorder keys that are not separated by empty lines.
+reorder_keys = false
+# Maximum amount of allowed consecutive blank lines. This does not affect the whitespace at the end of the document, as it is always stripped.
+allowed_blank_lines = 1
+# Use CRLF for line endings.
+crlf = false
+
+[[rule]]
+keys = [
+    "build-dependencies",
+    "dependencies",
+    "dev-dependencies",
+    "workspace.dependencies",
+]
+formatting = { reorder_keys = true }
+
+[[rule]]
+keys = ["package"]
+formatting = { reorder_keys = false }

Cargo.toml

@@ -15,10 +15,12 @@ readme = "README.md"
 repository = "https://github.com/datafusion-contrib/datafusion-federation"
 
 [workspace.dependencies]
-async-trait = "0.1.81"
+arrow = "53.0.0"
+arrow-flight = { version = "53.0.0", features = ["flight-sql-experimental"] }
+arrow-json = "53.0.0"
 async-stream = "0.3.5"
+async-trait = "0.1.81"
+datafusion = "42.0.0"
+datafusion-federation = { path = "datafusion-federation", version = "0.2.1" }
+datafusion-substrait = "42.0.0"
 futures = "0.3.30"
-datafusion = "41.0.0"
-datafusion-substrait = "41.0.0"
-arrow-json = "52.2.0"
-datafusion-federation = "0.2.1"

datafusion-federation/Cargo.toml

@@ -21,17 +21,16 @@ no-default-features = true
 sql = []
 
 [dependencies]
-futures.workspace = true
+arrow-json.workspace = true
+async-stream.workspace = true
 async-trait.workspace = true
 datafusion.workspace = true
-async-stream.workspace = true
-arrow-json.workspace = true
-
+futures.workspace = true
 
 [dev-dependencies]
 tokio = { version = "1.39.3", features = ["full"] }
-tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
 tracing = "0.1.40"
+tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
 
 [[example]]
 name = "df-csv"

datafusion-federation/src/sql/mod.rs

@@ -419,14 +419,6 @@ fn rewrite_table_scans_in_expr(
                 try_cast.data_type,
             )))
         }
-        Expr::Sort(sort) => {
-            let expr = rewrite_table_scans_in_expr(*sort.expr, known_rewrites)?;
-            Ok(Expr::Sort(Sort::new(
-                Box::new(expr),
-                sort.asc,
-                sort.nulls_first,
-            )))
-        }
         Expr::ScalarFunction(sf) => {
             let args = sf
                 .args
@@ -451,10 +443,13 @@ fn rewrite_table_scans_in_expr(
                 .map(Box::new);
             let order_by = af
                 .order_by
-                .map(|e| {
-                    e.into_iter()
-                        .map(|e| rewrite_table_scans_in_expr(e, known_rewrites))
-                        .collect::<Result<Vec<Expr>>>()
+                .map(|s| {
+                    s.into_iter()
+                        .map(|s| {
+                            rewrite_table_scans_in_expr(s.expr, known_rewrites)
+                                .map(|e| Sort::new(e, s.asc, s.nulls_first))
+                        })
+                        .collect::<Result<Vec<Sort>>>()
                 })
                 .transpose()?;
             Ok(Expr::AggregateFunction(AggregateFunction {
@@ -480,8 +475,11 @@ fn rewrite_table_scans_in_expr(
             let order_by = wf
                 .order_by
                 .into_iter()
-                .map(|e| rewrite_table_scans_in_expr(e, known_rewrites))
-                .collect::<Result<Vec<Expr>>>()?;
+                .map(|s| {
+                    rewrite_table_scans_in_expr(s.expr, known_rewrites)
+                        .map(|e| Sort::new(e, s.asc, s.nulls_first))
+                })
+                .collect::<Result<Vec<Sort>>>()?;
             Ok(Expr::WindowFunction(WindowFunction {
                 fun: wf.fun,
                 args,
@@ -533,13 +531,14 @@ fn rewrite_table_scans_in_expr(
                 is.negated,
             )))
         }
-        Expr::Wildcard { qualifier } => {
+        Expr::Wildcard { qualifier, options } => {
             if let Some(rewrite) = qualifier.as_ref().and_then(|q| known_rewrites.get(q)) {
                 Ok(Expr::Wildcard {
                     qualifier: Some(rewrite.clone()),
+                    options,
                 })
             } else {
-                Ok(Expr::Wildcard { qualifier })
+                Ok(Expr::Wildcard { qualifier, options })
             }
         }
         Expr::GroupingSet(gs) => match gs {

datafusion-federation/src/table_provider.rs

@@ -1,4 +1,4 @@
-use std::{any::Any, sync::Arc};
+use std::{any::Any, borrow::Cow, sync::Arc};
 
 use async_trait::async_trait;
 use datafusion::{
@@ -70,7 +70,7 @@ impl TableProvider for FederatedTableProviderAdaptor {
 
         self.source.table_type()
     }
-    fn get_logical_plan(&self) -> Option<&LogicalPlan> {
+    fn get_logical_plan(&self) -> Option<Cow<LogicalPlan>> {
         if let Some(table_provider) = &self.table_provider {
             return table_provider
                 .get_logical_plan()

datafusion-flight-sql-server/Cargo.toml

@@ -11,23 +11,23 @@ name = "datafusion_flight_sql_server"
 path = "src/lib.rs"
 
 [dependencies]
+arrow.workspace = true
+arrow-flight.workspace = true
 datafusion.workspace = true
-datafusion-substrait.workspace = true
 datafusion-federation = { workspace = true, features = ["sql"] }
+datafusion-substrait.workspace = true
 
 futures = "0.3.30"
-tonic = { version = "0.11.0", features = [
+log = "0.4.22"
+once_cell = "1.19.0"
+prost = "0.13.1"
+tonic = { version = "0.12.1", features = [
     "tls",
     "transport",
     "codegen",
     "prost",
 ] }
-prost = "0.12.3"
-arrow = "52.0.0"
-arrow-flight = { version = "52.2.0", features = ["flight-sql-experimental"] }
-log = "0.4.22"
-once_cell = "1.19.0"
 
 [dev-dependencies]
-tokio = { version = "1.39.3", features = ["full"] }
 datafusion-flight-sql-table-provider = { path = "../datafusion-flight-sql-table-provider" }
+tokio = { version = "1.39.3", features = ["full"] }

datafusion-flight-sql-table-provider/Cargo.toml

@@ -6,16 +6,16 @@ license.workspace = true
 readme.workspace = true
 
 [dependencies]
+arrow.workspace = true
+arrow-flight.workspace = true
 async-trait.workspace = true
 datafusion.workspace = true
 datafusion-federation = { workspace = true, features = ["sql"] }
 
 futures = "0.3.30"
-tonic = { version = "0.11.0", features = [
+tonic = { version = "0.12.1", features = [
     "tls",
     "transport",
     "codegen",
     "prost",
 ] }
-arrow = "52.0.0"
-arrow-flight = { version = "52.2.0", features = ["flight-sql-experimental"] }