From f076e96388fb038e3444b84719874d0c004ccf84 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 4 Jun 2024 18:10:01 -0600 Subject: [PATCH] save --- core/src/execution/datafusion/planner.rs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/core/src/execution/datafusion/planner.rs b/core/src/execution/datafusion/planner.rs index 6e1e68b61..dd476205b 100644 --- a/core/src/execution/datafusion/planner.rs +++ b/core/src/execution/datafusion/planner.rs @@ -731,8 +731,24 @@ impl PhysicalPlanner { // this approach looks like it has benefit then we may want to build a custom // version of CoalesceBatchesExec that copies when needed let copied = Arc::new(CopyExec::new(projection)); + + // keep this small to avoid error: + let target_batch_size = 1024; + + /* + org.apache.comet.CometNativeException: range end index 8504 out of range for slice of length 8192 + at comet::errors::init::{{closure}}(__internal__:0) + at std::panicking::rust_panic_with_hook(__internal__:0) + at std::panicking::begin_panic_handler::{{closure}}(__internal__:0) + at std::sys_common::backtrace::__rust_end_short_backtrace(__internal__:0) + at rust_begin_unwind(__internal__:0) + at core::panicking::panic_fmt(__internal__:0) + at core::slice::index::slice_end_index_len_fail(__internal__:0) + at comet::execution::datafusion::shuffle_writer::external_shuffle::{{closure}}(__internal__:0) + */ + let coalesced: Arc = - Arc::new(CoalesceBatchesExec::new(copied, 4196)); + Arc::new(CoalesceBatchesExec::new(copied, target_batch_size)); Ok((scans, coalesced)) } OpStruct::Filter(filter) => {