Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Remove use of nightly int_roundings feature #228

Merged
merged 1 commit into from
Mar 24, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions core/src/execution/datafusion/expressions/avg_decimal.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ use arrow_data::decimal::{
validate_decimal_precision, MAX_DECIMAL_FOR_EACH_PRECISION, MIN_DECIMAL_FOR_EACH_PRECISION,
};

use num::Integer;
use num::{integer::div_ceil, Integer};
use DataType::*;

/// AVG aggregate expression
Expand Down Expand Up @@ -514,7 +514,7 @@ fn avg(sum: i128, count: i128, target_min: i128, target_max: i128, scaler: i128)
if let Some(value) = sum.checked_mul(scaler) {
// `sum / count` with ROUND_HALF_UP
let (div, rem) = value.div_rem(&count);
let half = count.div_ceil(2);
let half = div_ceil(count, 2);
let half_neg = half.neg_wrapping();
let new_value = match value >= 0 {
true if rem >= half => div.add_wrapping(1),
Expand Down
9 changes: 6 additions & 3 deletions core/src/execution/datafusion/expressions/scalar_funcs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,10 @@ use datafusion_common::{
use datafusion_physical_expr::{
execution_props::ExecutionProps, functions::create_physical_fun, math_expressions,
};
use num::{BigInt, Signed, ToPrimitive};
use num::{
integer::{div_ceil, div_floor},
BigInt, Signed, ToPrimitive,
};
use unicode_segmentation::UnicodeSegmentation;

/// Create a physical scalar function.
Expand Down Expand Up @@ -249,13 +252,13 @@ fn long_to_decimal(v: &Option<i64>, precision: u8) -> Option<i128> {
#[inline]
fn decimal_ceil_f(scale: &i8) -> impl Fn(i128) -> i128 {
let div = 10_i128.pow_wrapping(*scale as u32);
move |x: i128| x.div_ceil(div)
move |x: i128| div_ceil(x, div)
}

#[inline]
fn decimal_floor_f(scale: &i8) -> impl Fn(i128) -> i128 {
let div = 10_i128.pow_wrapping(*scale as u32);
move |x: i128| x.div_floor(div)
move |x: i128| div_floor(x, div)
}

// Spark uses BigDecimal. See RoundBase implementation in Spark. Instead, we do the same by
Expand Down
5 changes: 3 additions & 2 deletions core/src/execution/datafusion/expressions/utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ use arrow_schema::DataType;
use chrono::{DateTime, Offset, TimeZone};
use datafusion_common::cast::as_generic_string_array;
use datafusion_physical_expr::PhysicalExpr;
use num::integer::div_floor;
use std::{any::Any, sync::Arc};

/// An utility function from DataFusion. It is not exposed by DataFusion.
Expand Down Expand Up @@ -198,13 +199,13 @@ pub(crate) fn spark_cast(array: ArrayRef, from_type: &DataType, to_type: &DataTy
match (from_type, to_type) {
(DataType::Timestamp(_, _), DataType::Int64) => {
// See Spark's `Cast` expression
unary_dyn::<_, Int64Type>(&array, |v| v.div_floor(MICROS_PER_SECOND)).unwrap()
unary_dyn::<_, Int64Type>(&array, |v| div_floor(v, MICROS_PER_SECOND)).unwrap()
}
(DataType::Dictionary(_, value_type), DataType::Int64)
if matches!(value_type.as_ref(), &DataType::Timestamp(_, _)) =>
{
// See Spark's `Cast` expression
unary_dyn::<_, Int64Type>(&array, |v| v.div_floor(MICROS_PER_SECOND)).unwrap()
unary_dyn::<_, Int64Type>(&array, |v| div_floor(v, MICROS_PER_SECOND)).unwrap()
}
(DataType::Timestamp(_, _), DataType::Utf8) => remove_trailing_zeroes(array),
(DataType::Dictionary(_, value_type), DataType::Utf8)
Expand Down
1 change: 0 additions & 1 deletion core/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
#![allow(clippy::upper_case_acronyms)]
#![allow(clippy::derive_partial_eq_without_eq)] // For prost generated struct
#![cfg_attr(feature = "nightly", feature(core_intrinsics))]
#![feature(int_roundings)]
#![feature(specialization)]

// Branch prediction hint. This is currently only available on nightly.
Expand Down
Loading