From 16d7e2caaa6b5c2d9183a11b029d7a4748560b46 Mon Sep 17 00:00:00 2001 From: psvri Date: Fri, 19 Apr 2024 18:13:52 +0530 Subject: [PATCH 1/2] Cleanup how datafusion session config is created --- core/src/execution/jni_api.rs | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/core/src/execution/jni_api.rs b/core/src/execution/jni_api.rs index 8249097a1..b5967a16e 100644 --- a/core/src/execution/jni_api.rs +++ b/core/src/execution/jni_api.rs @@ -216,18 +216,11 @@ fn prepare_datafusion_session_context( } } - // Get Datafusion configuration from Spark Execution context - // can be configured in Comet Spark JVM using Spark --conf parameters - // e.g: spark-shell --conf spark.datafusion.sql_parser.parse_float_as_decimal=true - let df_config = conf - .iter() - .filter(|(k, _)| k.starts_with("datafusion.")) - .map(|kv| (kv.0.to_owned(), kv.1.to_owned())) - .collect::>(); - - let session_config = - SessionConfig::from_string_hash_map(std::collections::HashMap::from_iter(df_config))? - .with_batch_size(batch_size); + let mut session_config = SessionConfig::new().with_batch_size(batch_size); + + for (key, value) in conf.iter().filter(|(k, _)| k.starts_with("datafusion.")) { + session_config = session_config.set_str(key, value); + } let runtime = RuntimeEnv::new(rt_config).unwrap(); From dd573e2354e47f1c6bb667a3942c617a08c1afc8 Mon Sep 17 00:00:00 2001 From: psvri Date: Sat, 20 Apr 2024 00:29:06 +0530 Subject: [PATCH 2/2] Readd the comments --- core/src/execution/jni_api.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/src/execution/jni_api.rs b/core/src/execution/jni_api.rs index b5967a16e..bc194238b 100644 --- a/core/src/execution/jni_api.rs +++ b/core/src/execution/jni_api.rs @@ -216,6 +216,9 @@ fn prepare_datafusion_session_context( } } + // Get Datafusion configuration from Spark Execution context + // can be configured in Comet Spark JVM using Spark --conf parameters + // e.g: spark-shell --conf spark.datafusion.sql_parser.parse_float_as_decimal=true let mut session_config = SessionConfig::new().with_batch_size(batch_size); for (key, value) in conf.iter().filter(|(k, _)| k.starts_with("datafusion.")) {