update to datafusion 42 ... (apache#1080)

... and all other dependant libraries
milenkovicm · Oct 14, 2024 · 1c4a38c · 1c4a38c
1 parent 5c7f4af
commit 1c4a38c
Show file tree

Hide file tree

Showing 19 changed files with 306 additions and 391 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -21,23 +21,23 @@ members = ["ballista-cli", "ballista/client", "ballista/core", "ballista/executo
 resolver = "2"
 
 [workspace.dependencies]
-arrow = { version = "52.2.0", features = ["ipc_compression"] }
-arrow-flight = { version = "52.2.0", features = ["flight-sql-experimental"] }
-arrow-schema = { version = "52.2.0", default-features = false }
+arrow = { version = "53", features = ["ipc_compression"] }
+arrow-flight = { version = "53", features = ["flight-sql-experimental"] }
+arrow-schema = { version = "53", default-features = false }
 clap = { version = "3", features = ["derive", "cargo"] }
 configure_me = { version = "0.4.0" }
 configure_me_codegen = { version = "0.4.4" }
 # bump directly to datafusion v43 to avoid the serde bug on v42 (https://github.com/apache/datafusion/pull/12626)
-datafusion = "41.0.0"
-datafusion-cli = "41.0.0"
-datafusion-proto = "41.0.0"
-datafusion-proto-common = "41.0.0"
-object_store = "0.10.2"
-prost = "0.12.0"
-prost-types = "0.12.0"
-sqlparser = "0.49.0"
-tonic = { version = "0.11.0" }
-tonic-build = { version = "0.11.0", default-features = false, features = [
+datafusion = "42.0.0"
+datafusion-cli = "42.0.0"
+datafusion-proto = "42.0.0"
+datafusion-proto-common = "42.0.0"
+object_store = "0.11"
+prost = "0.13"
+prost-types = "0.13"
+sqlparser = "0.50"
+tonic = { version = "0.12" }
+tonic-build = { version = "0.12", default-features = false, features = [
     "transport",
     "prost"
 ] }

diff --git a/ballista-cli/Cargo.toml b/ballista-cli/Cargo.toml
@@ -30,14 +30,15 @@ readme = "README.md"
 
 [dependencies]
 ballista = { path = "../ballista/client", version = "0.12.0", features = ["standalone"] }
-clap = { workspace = true }
+# datafusion-cli uses 4.5 clap, thus it does not depend on workspace
+clap = { version = "4.5", features = ["derive", "cargo"] }
 datafusion = { workspace = true }
 datafusion-cli = { workspace = true }
 dirs = "5.0.1"
 env_logger = { workspace = true }
 mimalloc = { version = "0.1", default-features = false }
 num_cpus = { workspace = true }
-rustyline = "11.0.0"
+rustyline = "14.0.0"
 tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot"] }
 
 [features]

diff --git a/ballista-cli/src/command.rs b/ballista-cli/src/command.rs
@@ -22,7 +22,7 @@ use std::sync::Arc;
 use std::time::Instant;
 
 use ballista::prelude::{BallistaContext, BallistaError, Result};
-use clap::ArgEnum;
+
 use datafusion::arrow::array::{ArrayRef, StringArray};
 use datafusion::arrow::datatypes::{DataType, Field, Schema};
 use datafusion::arrow::record_batch::RecordBatch;
@@ -223,7 +223,7 @@ impl OutputFormat {
                     Err(BallistaError::General(format!(
                         "{:?} is not a valid format type [possible values: {:?}]",
                         format,
-                        PrintFormat::value_variants()
+                        "TO BE FIXED", //PrintFormat::value_variants()
                     )))
                 }
             }

diff --git a/ballista-cli/src/main.rs b/ballista-cli/src/main.rs
@@ -58,7 +58,7 @@ struct Args {
     #[clap(
         short,
         long,
-        multiple_values = true,
+        num_args = 0..,
         help = "Execute commands from file(s), then exit",
         value_parser(parse_valid_file)
     )]
@@ -67,7 +67,7 @@ struct Args {
     #[clap(
         short = 'r',
         long,
-        multiple_values = true,
+        num_args = 0..,
         help = "Run the provided files on startup instead of ~/.ballistarc",
         value_parser(parse_valid_file),
         conflicts_with = "file"

diff --git a/ballista/core/build.rs b/ballista/core/build.rs
@@ -44,7 +44,7 @@ fn main() -> Result<(), String> {
             .extern_path(".datafusion_common", "::datafusion_proto_common")
             .extern_path(".datafusion", "::datafusion_proto::protobuf")
             .protoc_arg("--experimental_allow_proto3_optional")
-            .compile(&["proto/ballista.proto"], &["proto"])
+            .compile_protos(&["proto/ballista.proto"], &["proto"])
             .map_err(|e| format!("protobuf compilation failed: {e}"))?;
         let generated_source_path = out.join("ballista.protobuf.rs");
         let code = std::fs::read_to_string(generated_source_path).unwrap();