From c3432bea7fc202f86daa727125a8f4db725255c6 Mon Sep 17 00:00:00 2001 From: Vrishabh Date: Sun, 21 Apr 2024 02:04:49 +0530 Subject: [PATCH] Chore: Cleanup how datafusion session config is created (#289) --- core/src/execution/jni_api.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/core/src/execution/jni_api.rs b/core/src/execution/jni_api.rs index 8249097a18..bc194238ba 100644 --- a/core/src/execution/jni_api.rs +++ b/core/src/execution/jni_api.rs @@ -219,15 +219,11 @@ fn prepare_datafusion_session_context( // Get Datafusion configuration from Spark Execution context // can be configured in Comet Spark JVM using Spark --conf parameters // e.g: spark-shell --conf spark.datafusion.sql_parser.parse_float_as_decimal=true - let df_config = conf - .iter() - .filter(|(k, _)| k.starts_with("datafusion.")) - .map(|kv| (kv.0.to_owned(), kv.1.to_owned())) - .collect::>(); - - let session_config = - SessionConfig::from_string_hash_map(std::collections::HashMap::from_iter(df_config))? - .with_batch_size(batch_size); + let mut session_config = SessionConfig::new().with_batch_size(batch_size); + + for (key, value) in conf.iter().filter(|(k, _)| k.starts_with("datafusion.")) { + session_config = session_config.set_str(key, value); + } let runtime = RuntimeEnv::new(rt_config).unwrap();