From 16d7e2caaa6b5c2d9183a11b029d7a4748560b46 Mon Sep 17 00:00:00 2001 From: psvri Date: Fri, 19 Apr 2024 18:13:52 +0530 Subject: [PATCH] Cleanup how datafusion session config is created --- core/src/execution/jni_api.rs | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/core/src/execution/jni_api.rs b/core/src/execution/jni_api.rs index 8249097a1..b5967a16e 100644 --- a/core/src/execution/jni_api.rs +++ b/core/src/execution/jni_api.rs @@ -216,18 +216,11 @@ fn prepare_datafusion_session_context( } } - // Get Datafusion configuration from Spark Execution context - // can be configured in Comet Spark JVM using Spark --conf parameters - // e.g: spark-shell --conf spark.datafusion.sql_parser.parse_float_as_decimal=true - let df_config = conf - .iter() - .filter(|(k, _)| k.starts_with("datafusion.")) - .map(|kv| (kv.0.to_owned(), kv.1.to_owned())) - .collect::>(); - - let session_config = - SessionConfig::from_string_hash_map(std::collections::HashMap::from_iter(df_config))? - .with_batch_size(batch_size); + let mut session_config = SessionConfig::new().with_batch_size(batch_size); + + for (key, value) in conf.iter().filter(|(k, _)| k.starts_with("datafusion.")) { + session_config = session_config.set_str(key, value); + } let runtime = RuntimeEnv::new(rt_config).unwrap();