From 2036ddc3854f410f7fa4138ec7c2c4cd7d21e0e2 Mon Sep 17 00:00:00 2001 From: dantengsky Date: Fri, 22 Sep 2023 18:57:01 +0800 Subject: [PATCH] wip --- src/query/storages/common/blocks/src/block.rs | 33 +++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/src/query/storages/common/blocks/src/block.rs b/src/query/storages/common/blocks/src/block.rs index eb37b633491fe..7261a6505189c 100644 --- a/src/query/storages/common/blocks/src/block.rs +++ b/src/query/storages/common/blocks/src/block.rs @@ -14,6 +14,7 @@ use common_arrow::arrow::chunk::Chunk; use common_arrow::arrow::datatypes::DataType as ArrowDataType; +use common_arrow::arrow::datatypes::DataType; use common_arrow::arrow::io::parquet::write::transverse; use common_arrow::arrow::io::parquet::write::RowGroupIterator; use common_arrow::arrow::io::parquet::write::WriteOptions; @@ -92,19 +93,25 @@ pub fn blocks_to_parquet( fn col_encoding(data_type: &ArrowDataType) -> Encoding { // - match data_type { - ArrowDataType::Int8 - | ArrowDataType::Int16 - | ArrowDataType::Int32 - | ArrowDataType::Int64 - | ArrowDataType::UInt8 - | ArrowDataType::UInt16 - | ArrowDataType::UInt32 - | ArrowDataType::UInt64 => Encoding::DeltaBinaryPacked, - ArrowDataType::Float16 | ArrowDataType::Float32 => Encoding::ByteStreamSplit, - // | ArrowDataType::LargeBinary - // | ArrowDataType::Utf8 - // | ArrowDataType::LargeUtf8 => Encoding::DeltaLengthByteArray, + let lt = data_type.to_logical_type(); + match lt { + DataType::Decimal(p, _) if *p <= 18 => Encoding::DeltaBinaryPacked, + + DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + | DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::Date32 + | DataType::Time32(_) + | DataType::Int64 + | DataType::Date64 + | DataType::Time64(_) + | DataType::Timestamp(_, _) + | DataType::Duration(_) => Encoding::DeltaBinaryPacked, + _ => Encoding::Plain, } }