Skip to content

Commit

Permalink
chore(query): using Bitmap::new_constant to replace constant_bitmap aโ€ฆ
Browse files Browse the repository at this point in the history 
โ€ฆnd extend_constant (#13309)

* use new_constant

* update dependence
  • Loading branch information
@Dousir9
Dousir9 authored Oct 18, 2023
1 parent c22249d commit 6262730
Show file tree
Hide file tree
Showing 17 changed files with 51 additions and 91 deletions.
2 changes: 1 addition & 1 deletion Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,7 @@ rpath = false
[patch.crates-io]
# If there are dependencies that need patching, they can be listed below.

arrow2 = { git = "https://github.com/jorgecarleitao/arrow2", rev = "dd80c89" }
arrow2 = { git = "https://github.com/jorgecarleitao/arrow2", rev = "3c61372" }
arrow-format = { git = "https://github.com/everpcpc/arrow-format", rev = "588d371" }
parquet2 = { git = "https://github.com/jorgecarleitao/parquet2", rev = "b0e6545" }
metrics = { git = "https://github.com/datafuse-extras/metrics.git", rev = "fc2ecd1" }
Expand Down
38 changes: 14 additions & 24 deletions src/query/expression/src/evaluator.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@ use crate::types::BooleanType;
use crate::types::DataType;
use crate::types::NullableType;
use crate::udf_client::UDFFlightClient;
use crate::utils::arrow::constant_bitmap;
use crate::utils::variant_transform::contains_variant;
use crate::utils::variant_transform::transform_variant;
use crate::values::Column;
Expand Down Expand Up @@ -420,7 +419,7 @@ impl<'a> Evaluator<'a> {
.into_column()
.unwrap();
Ok(Value::Column(Column::Nullable(Box::new(NullableColumn {
validity: constant_bitmap(true, column.len()).into(),
validity: Bitmap::new_constant(true, column.len()),
column,
}))))
}
Expand All @@ -442,11 +441,8 @@ impl<'a> Evaluator<'a> {
},
(DataType::Array(inner_src_ty), DataType::Array(inner_dest_ty)) => match value {
Value::Scalar(Scalar::Array(array)) => {
let validity = validity.map(|validity| {
let mut inner_validity = MutableBitmap::with_capacity(array.len());
inner_validity.extend_constant(array.len(), validity.get_bit(0));
inner_validity.into()
});
let validity = validity
.map(|validity| Bitmap::new_constant(validity.get_bit(0), array.len()));

let new_array = self
.run_cast(
Expand Down Expand Up @@ -505,11 +501,8 @@ impl<'a> Evaluator<'a> {
},
(DataType::Map(inner_src_ty), DataType::Map(inner_dest_ty)) => match value {
Value::Scalar(Scalar::Map(array)) => {
let validity = validity.map(|validity| {
let mut inner_validity = MutableBitmap::with_capacity(array.len());
inner_validity.extend_constant(array.len(), validity.get_bit(0));
inner_validity.into()
});
let validity = validity
.map(|validity| Bitmap::new_constant(validity.get_bit(0), array.len()));

let new_array = self
.run_cast(
Expand All @@ -524,11 +517,8 @@ impl<'a> Evaluator<'a> {
Ok(Value::Scalar(Scalar::Map(new_array)))
}
Value::Column(Column::Map(col)) => {
let validity = validity.map(|validity| {
let mut inner_validity = MutableBitmap::with_capacity(col.len());
inner_validity.extend_constant(col.len(), validity.get_bit(0));
inner_validity.into()
});
let validity = validity
.map(|validity| Bitmap::new_constant(validity.get_bit(0), col.len()));

let new_col = self
.run_cast(
Expand Down Expand Up @@ -654,7 +644,7 @@ impl<'a> Evaluator<'a> {
Value::Scalar(_) => Ok(value),
Value::Column(column) => {
Ok(Value::Column(Column::Nullable(Box::new(NullableColumn {
validity: constant_bitmap(true, column.len()).into(),
validity: Bitmap::new_constant(true, column.len()),
column,
}))))
}
Expand Down Expand Up @@ -692,7 +682,7 @@ impl<'a> Evaluator<'a> {
offsets: col.offsets,
}));
Ok(Value::Column(Column::Nullable(Box::new(NullableColumn {
validity: constant_bitmap(true, new_col.len()).into(),
validity: Bitmap::new_constant(true, new_col.len()),
column: new_col,
}))))
}
Expand Down Expand Up @@ -730,7 +720,7 @@ impl<'a> Evaluator<'a> {
offsets: col.offsets,
}));
Ok(Value::Column(Column::Nullable(Box::new(NullableColumn {
validity: constant_bitmap(true, new_col.len()).into(),
validity: Bitmap::new_constant(true, new_col.len()),
column: new_col,
}))))
}
Expand Down Expand Up @@ -848,7 +838,7 @@ impl<'a> Evaluator<'a> {
});

// Evaluate the condition first and then partially evaluate the result branches.
let mut validity = validity.unwrap_or_else(|| constant_bitmap(true, num_rows).into());
let mut validity = validity.unwrap_or_else(|| Bitmap::new_constant(true, num_rows));
let mut conds = Vec::new();
let mut flags = Vec::new();
let mut results = Vec::new();
Expand All @@ -857,12 +847,12 @@ impl<'a> Evaluator<'a> {
match cond.try_downcast::<NullableType<BooleanType>>().unwrap() {
Value::Scalar(None | Some(false)) => {
results.push(Value::Scalar(Scalar::default_value(&generics[0])));
flags.push(constant_bitmap(false, len.unwrap_or(1)).into());
flags.push(Bitmap::new_constant(false, len.unwrap_or(1)));
}
Value::Scalar(Some(true)) => {
results.push(self.partial_run(&args[cond_idx + 1], Some(validity.clone()))?);
validity = constant_bitmap(false, num_rows).into();
flags.push(constant_bitmap(true, len.unwrap_or(1)).into());
validity = Bitmap::new_constant(false, num_rows);
flags.push(Bitmap::new_constant(true, len.unwrap_or(1)));
break;
}
Value::Column(cond) => {
Expand Down
9 changes: 4 additions & 5 deletions src/query/expression/src/function.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ use crate::type_check::try_unify_signature;
use crate::types::nullable::NullableColumn;
use crate::types::nullable::NullableDomain;
use crate::types::*;
use crate::utils::arrow::constant_bitmap;
use crate::values::Value;
use crate::values::ValueRef;
use crate::Column;
Expand Down Expand Up @@ -258,7 +257,7 @@ impl Function {
Value::Column(column) => {
Value::Column(Column::Nullable(Box::new(NullableColumn {
column,
validity: constant_bitmap(true, num_rows).into(),
validity: Bitmap::new_constant(true, num_rows),
})))
}
}
Expand Down Expand Up @@ -550,7 +549,7 @@ impl<'a> EvalContext<'a> {
valids.set(row, false);
}
None => {
let mut valids = constant_bitmap(true, self.num_rows.max(1));
let mut valids = Bitmap::new_constant(true, self.num_rows.max(1)).make_mut();
valids.set(row, false);
self.errors = Some((valids, error_msg.into()));
}
Expand Down Expand Up @@ -628,7 +627,7 @@ where F: Fn(&[ValueRef<AnyType>], &mut EvalContext) -> Value<AnyType> {
}
}
let results = f(&nonull_args, ctx);
let bitmap = bitmap.unwrap_or_else(|| constant_bitmap(true, len));
let bitmap = bitmap.unwrap_or_else(|| Bitmap::new_constant(true, len).make_mut());
if let Some((error_bitmap, _)) = ctx.errors.as_mut() {
// If the original value is NULL, we can ignore the error.
let rhs: Bitmap = bitmap.clone().not().into();
Expand Down Expand Up @@ -690,7 +689,7 @@ pub fn error_to_null<I1: ArgType, O: ArgType>(
Value::Scalar(scalar) => Value::Scalar(Some(scalar)),
Value::Column(column) => Value::Column(NullableColumn {
column,
validity: constant_bitmap(true, ctx.num_rows).into(),
validity: Bitmap::new_constant(true, ctx.num_rows),
}),
}
}
Expand Down
6 changes: 0 additions & 6 deletions src/query/expression/src/utils/arrow.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,12 +53,6 @@ pub fn append_bitmap(bitmap: &mut MutableBitmap, other: &Bitmap) {
bitmap.extend_from_bitmap(other)
}

pub fn constant_bitmap(value: bool, len: usize) -> MutableBitmap {
let mut builder = MutableBitmap::new();
builder.extend_constant(len, value);
builder
}

pub fn buffer_into_mut<T: Clone>(mut buffer: Buffer<T>) -> Vec<T> {
unsafe {
buffer
Expand Down
22 changes: 8 additions & 14 deletions src/query/expression/src/values.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,6 @@ use crate::types::*;
use crate::utils::arrow::append_bitmap;
use crate::utils::arrow::bitmap_into_mut;
use crate::utils::arrow::buffer_into_mut;
use crate::utils::arrow::constant_bitmap;
use crate::utils::arrow::deserialize_column;
use crate::utils::arrow::serialize_column;
use crate::utils::FromData;
Expand Down Expand Up @@ -1626,11 +1625,10 @@ impl Column {
};

if is_nullable {
let validity = arrow_col.validity().cloned().unwrap_or_else(|| {
let mut validity = MutableBitmap::with_capacity(arrow_col.len());
validity.extend_constant(arrow_col.len(), true);
validity.into()
});
let validity = arrow_col
.validity()
.cloned()
.unwrap_or_else(|| Bitmap::new_constant(true, arrow_col.len()));
Column::Nullable(Box::new(NullableColumn { column, validity }))
} else {
column
Expand Down Expand Up @@ -1774,11 +1772,7 @@ impl Column {
}))
}
_ => {
let validity = validity.unwrap_or_else(|| {
let mut validity = MutableBitmap::with_capacity(self.len());
validity.extend_constant(self.len(), true);
validity.into()
});
let validity = validity.unwrap_or_else(|| Bitmap::new_constant(true, self.len()));
Column::Nullable(Box::new(NullableColumn {
column: self.clone(),
validity,
Expand Down Expand Up @@ -1935,7 +1929,7 @@ impl ColumnBuilder {
}
return ColumnBuilder::Nullable(Box::new(NullableColumnBuilder {
builder,
validity: constant_bitmap(true, n),
validity: Bitmap::new_constant(true, n).make_mut(),
}));
}
}
Expand All @@ -1950,7 +1944,7 @@ impl ColumnBuilder {
}
ColumnBuilder::Nullable(Box::new(NullableColumnBuilder {
builder,
validity: constant_bitmap(false, n),
validity: Bitmap::new_constant(false, n).make_mut(),
}))
}
_ => unreachable!(),
Expand All @@ -1961,7 +1955,7 @@ impl ColumnBuilder {
ScalarRef::Decimal(dec) => {
ColumnBuilder::Decimal(DecimalColumnBuilder::repeat(*dec, n))
}
ScalarRef::Boolean(b) => ColumnBuilder::Boolean(constant_bitmap(*b, n)),
ScalarRef::Boolean(b) => ColumnBuilder::Boolean(Bitmap::new_constant(*b, n).make_mut()),
ScalarRef::String(s) => ColumnBuilder::String(StringColumnBuilder::repeat(s, n)),
ScalarRef::Timestamp(d) => ColumnBuilder::Timestamp(vec![*d; n]),
ScalarRef::Date(d) => ColumnBuilder::Date(vec![*d; n]),
Expand Down
7 changes: 3 additions & 4 deletions src/query/formats/tests/it/output_format_utils.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

use common_arrow::arrow::bitmap::MutableBitmap;
use common_arrow::arrow::bitmap::Bitmap;
use common_expression::types::nullable::NullableColumn;
use common_expression::types::number::Float64Type;
use common_expression::types::number::Int32Type;
Expand Down Expand Up @@ -72,12 +72,11 @@ pub fn get_simple_block(is_nullable: bool) -> (TableSchemaRef, DataBlock) {
.into_iter()
.enumerate()
.map(|(idx, (data_type, c))| {
let mut validity = MutableBitmap::new();
validity.extend_constant(c.len(), true);
let validity = Bitmap::new_constant(true, c.len());
(
Column::Nullable(Box::new(NullableColumn {
column: c,
validity: validity.into(),
validity,
})),
TableField::new(&format!("c{}", idx + 1), data_type.wrap_nullable()),
)
Expand Down
4 changes: 2 additions & 2 deletions src/query/functions/src/scalars/arithmetic.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ use std::ops::BitOr;
use std::ops::BitXor;
use std::sync::Arc;

use common_arrow::arrow::bitmap::Bitmap;
use common_expression::types::decimal::Decimal;
use common_expression::types::decimal::DecimalColumn;
use common_expression::types::decimal::DecimalDomain;
Expand All @@ -44,7 +45,6 @@ use common_expression::types::ALL_NUMERICS_TYPES;
use common_expression::types::ALL_UNSIGNED_INTEGER_TYPES;
use common_expression::utils::arithmetics_type::ResultTypeOfBinary;
use common_expression::utils::arithmetics_type::ResultTypeOfUnary;
use common_expression::utils::arrow::constant_bitmap;
use common_expression::values::Value;
use common_expression::values::ValueRef;
use common_expression::vectorize_1_arg;
Expand Down Expand Up @@ -885,7 +885,7 @@ pub fn register_number_to_string(registry: &mut FunctionRegistry) {
let result = builder.build();
Value::Column(NullableColumn {
column: result,
validity: constant_bitmap(true, from.len()).into(),
validity: Bitmap::new_constant(true, from.len()),
})
}
},
Expand Down
6 changes: 3 additions & 3 deletions src/query/functions/src/scalars/datetime.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ use chrono::prelude::*;
use chrono::Datelike;
use chrono::Utc;
use chrono_tz::Tz;
use common_arrow::arrow::bitmap::Bitmap;
use common_arrow::arrow::temporal_conversions::EPOCH_DAYS_FROM_CE;
use common_expression::error_to_null;
use common_expression::types::date::check_date;
Expand Down Expand Up @@ -45,7 +46,6 @@ use common_expression::types::NullableType;
use common_expression::types::NumberType;
use common_expression::types::StringType;
use common_expression::types::TimestampType;
use common_expression::utils::arrow::constant_bitmap;
use common_expression::utils::date_helper::*;
use common_expression::vectorize_1_arg;
use common_expression::vectorize_2_arg;
Expand Down Expand Up @@ -531,7 +531,7 @@ fn register_to_number(registry: &mut FunctionRegistry) {
|val, _| match val {
ValueRef::Scalar(scalar) => Value::Scalar(Some(scalar as i64)),
ValueRef::Column(col) => Value::Column(NullableColumn {
validity: constant_bitmap(true, col.len()).into(),
validity: Bitmap::new_constant(true, col.len()),
column: col.iter().map(|val| *val as i64).collect(),
}),
},
Expand All @@ -548,7 +548,7 @@ fn register_to_number(registry: &mut FunctionRegistry) {
|val, _| match val {
ValueRef::Scalar(scalar) => Value::Scalar(Some(scalar)),
ValueRef::Column(col) => Value::Column(NullableColumn {
validity: constant_bitmap(true, col.len()).into(),
validity: Bitmap::new_constant(true, col.len()),
column: col,
}),
},
Expand Down
4 changes: 2 additions & 2 deletions src/query/functions/src/scalars/variant.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ use std::sync::Arc;

use bstr::ByteSlice;
use chrono::Datelike;
use common_arrow::arrow::bitmap::Bitmap;
use common_arrow::arrow::temporal_conversions::EPOCH_DAYS_FROM_CE;
use common_expression::types::date::string_to_date;
use common_expression::types::nullable::NullableColumn;
Expand All @@ -38,7 +39,6 @@ use common_expression::types::StringType;
use common_expression::types::TimestampType;
use common_expression::types::VariantType;
use common_expression::types::ALL_NUMERICS_TYPES;
use common_expression::utils::arrow::constant_bitmap;
use common_expression::vectorize_1_arg;
use common_expression::vectorize_with_builder_1_arg;
use common_expression::vectorize_with_builder_2_arg;
Expand Down Expand Up @@ -649,7 +649,7 @@ pub fn register(registry: &mut FunctionRegistry) {
ValueRef::Column(col) => {
let new_col = cast_scalars_to_variants(col.iter(), ctx.func_ctx.tz);
Value::Column(NullableColumn {
validity: constant_bitmap(true, new_col.len()).into(),
validity: Bitmap::new_constant(true, new_col.len()),
column: new_col,
})
}
Expand Down
11 changes: 3 additions & 8 deletions src/query/service/src/pipelines/processors/transforms/hash_join/common.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
use common_arrow::arrow::bitmap::Bitmap;
use common_arrow::arrow::bitmap::MutableBitmap;
use common_exception::Result;
use common_expression::arrow::constant_bitmap;
use common_expression::arrow::or_validities;
use common_expression::types::nullable::NullableColumn;
use common_expression::types::AnyType;
Expand Down Expand Up @@ -132,7 +131,7 @@ impl HashJoinProbeState {
match filter_vector {
Column::Nullable(_) => Ok(filter_vector),
other => Ok(Column::Nullable(Box::new(NullableColumn {
validity: constant_bitmap(true, other.len()).into(),
validity: Bitmap::new_constant(true, other.len()),
column: other,
}))),
}
Expand All @@ -156,19 +155,15 @@ impl HashJoinState {
Column::Nullable(c) => {
let bitmap = &c.validity;
if bitmap.unset_bits() == 0 {
let mut m = MutableBitmap::with_capacity(num_rows);
m.extend_constant(num_rows, true);
valids = Some(m.into());
valids = Some(Bitmap::new_constant(true, num_rows));
break;
} else {
valids = or_validities(valids, Some(bitmap.clone()));
}
}
Column::Null { .. } => {}
_c => {
let mut m = MutableBitmap::with_capacity(num_rows);
m.extend_constant(num_rows, true);
valids = Some(m.into());
valids = Some(Bitmap::new_constant(true, num_rows));
break;
}
}
Expand Down
Loading

1 comment on commit 6262730

@vercel
Copy link

@vercel vercel bot commented on 6262730 Oct 18, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Successfully deployed to the following URLs:

databend โ€“ ./

databend-git-main-databend.vercel.app
databend.vercel.app
databend-databend.vercel.app
databend.rs
databend.org

Please sign in to comment.