From e19d4c10779767af9bcbadfaa972be834f1cf6fe Mon Sep 17 00:00:00 2001 From: sundyli <543950155@qq.com> Date: Sun, 17 Dec 2023 18:09:45 -0800 Subject: [PATCH 01/20] feat(query): add decimal round/truncate/ceil/floor function (#14040) * feat(query): add decimal round/truncate function * chore(query): add tests * chore(query): add tests * chore(query): add tests * chore(query): add tests * fix(function): fix ceil function * chore(query): fix tests --- src/query/expression/src/type_check.rs | 29 + src/query/functions/src/scalars/decimal.rs | 270 ++++++++ src/query/functions/src/scalars/math.rs | 4 + .../it/scalars/testdata/function_list.txt | 208 +++--- .../tests/it/scalars/testdata/math.txt | 90 ++- src/tests/sqlsmith/src/sql_gen/func.rs | 4 +- .../02_function/02_0014_function_maths.test | 22 + tests/sqllogictests/suites/tpcds/queries.test | 628 +++++++++--------- tests/sqllogictests/suites/tpch/queries.test | 50 +- 9 files changed, 810 insertions(+), 495 deletions(-) diff --git a/src/query/expression/src/type_check.rs b/src/query/expression/src/type_check.rs index a30888662ca0..d1b1f887cb92 100755 --- a/src/query/expression/src/type_check.rs +++ b/src/query/expression/src/type_check.rs @@ -28,9 +28,12 @@ use crate::function::FunctionSignature; use crate::types::decimal::DecimalSize; use crate::types::decimal::MAX_DECIMAL128_PRECISION; use crate::types::decimal::MAX_DECIMAL256_PRECISION; +use crate::types::ArgType; use crate::types::DataType; use crate::types::DecimalDataType; +use crate::types::Int64Type; use crate::types::Number; +use crate::types::NumberScalar; use crate::AutoCastRules; use crate::ColumnIndex; use crate::ConstantFolder; @@ -124,6 +127,32 @@ pub fn check( } } + // inject the params + if ["round", "truncate"].contains(&name.as_str()) && params.is_empty() { + let mut scale = 0; + let mut new_args = args_expr.clone(); + + if args_expr.len() == 2 { + let scalar_expr = &args_expr[1]; + scale = check_number::<_, i64>( + scalar_expr.span(), + &FunctionContext::default(), + scalar_expr, + fn_registry, + )?; + } else { + new_args.push(Expr::Constant { + span: None, + scalar: Scalar::Number(NumberScalar::Int64(scale)), + data_type: Int64Type::data_type(), + }) + } + scale = scale.clamp(-76, 76); + let add_on_scale = (scale + 76) as usize; + let params = vec![add_on_scale]; + return check_function(*span, name, ¶ms, &args_expr, fn_registry); + } + check_function(*span, name, params, &args_expr, fn_registry) } RawExpr::LambdaFunctionCall { diff --git a/src/query/functions/src/scalars/decimal.rs b/src/query/functions/src/scalars/decimal.rs index 720fc7eced16..78618cc8720e 100644 --- a/src/query/functions/src/scalars/decimal.rs +++ b/src/query/functions/src/scalars/decimal.rs @@ -1639,3 +1639,273 @@ fn decimal_to_int( Value::Column(result) } } + +pub fn register_decimal_math(registry: &mut FunctionRegistry) { + let factory = |params: &[usize], args_type: &[DataType], round_mode: RoundMode| { + if args_type.is_empty() { + return None; + } + + let from_type = args_type[0].remove_nullable(); + if !matches!(from_type, DataType::Decimal(_)) { + return None; + } + + let from_decimal_type = from_type.as_decimal().unwrap(); + + let scale = if params.is_empty() { + 0 + } else { + params[0] as i64 - 76 + }; + + let decimal_size = DecimalSize { + precision: from_decimal_type.precision(), + scale: scale.clamp(0, from_decimal_type.scale() as i64) as u8, + }; + + let dest_decimal_type = DecimalDataType::from_size(decimal_size).ok()?; + let name = format!("{:?}", round_mode).to_lowercase(); + + let mut sig_args_type = args_type.to_owned(); + sig_args_type[0] = from_type.clone(); + let f = Function { + signature: FunctionSignature { + name, + args_type: sig_args_type, + return_type: DataType::Decimal(dest_decimal_type), + }, + eval: FunctionEval::Scalar { + calc_domain: Box::new(move |_ctx, _d| FunctionDomain::Full), + eval: Box::new(move |args, _ctx| { + decimal_round_truncate( + &args[0], + from_type.clone(), + dest_decimal_type, + scale, + round_mode, + ) + }), + }, + }; + + if args_type[0].is_nullable() { + Some(f.passthrough_nullable()) + } else { + Some(f) + } + }; + + for m in [ + RoundMode::Round, + RoundMode::Truncate, + RoundMode::Ceil, + RoundMode::Floor, + ] { + let name = format!("{:?}", m).to_lowercase(); + registry.register_function_factory(&name, move |params, args_type| { + Some(Arc::new(factory(params, args_type, m)?)) + }); + } +} + +#[derive(Copy, Clone, Debug)] +enum RoundMode { + Round, + Truncate, + Floor, + Ceil, +} + +fn decimal_round_positive(values: &[T], source_scale: i64, target_scale: i64) -> Vec +where T: Decimal + From + DivAssign + Div + Add + Sub { + let power_of_ten = T::e((source_scale - target_scale) as u32); + let addition = power_of_ten / T::from(2); + + values + .iter() + .map(|input| { + let input = if input < &T::zero() { + *input - addition + } else { + *input + addition + }; + input / power_of_ten + }) + .collect() +} + +fn decimal_round_negative(values: &[T], source_scale: i64, target_scale: i64) -> Vec +where T: Decimal + + From + + DivAssign + + Div + + Add + + Sub + + Mul { + let divide_power_of_ten = T::e((source_scale - target_scale) as u32); + let addition = divide_power_of_ten / T::from(2); + let multiply_power_of_ten = T::e((-target_scale) as u32); + + values + .iter() + .map(|input| { + let input = if input < &T::zero() { + *input - addition + } else { + *input + addition + }; + input / divide_power_of_ten * multiply_power_of_ten + }) + .collect() +} + +// if round mode is ceil, truncate should add one value +fn decimal_truncate_positive(values: &[T], source_scale: i64, target_scale: i64) -> Vec +where T: Decimal + From + DivAssign + Div + Add + Sub { + let power_of_ten = T::e((source_scale - target_scale) as u32); + + values.iter().map(|input| *input / power_of_ten).collect() +} + +fn decimal_truncate_negative(values: &[T], source_scale: i64, target_scale: i64) -> Vec +where T: Decimal + + From + + DivAssign + + Div + + Add + + Sub + + Mul { + let divide_power_of_ten = T::e((source_scale - target_scale) as u32); + let multiply_power_of_ten = T::e((-target_scale) as u32); + + values + .iter() + .map(|input| *input / divide_power_of_ten * multiply_power_of_ten) + .collect() +} + +fn decimal_floor(values: &[T], source_scale: i64) -> Vec +where T: Decimal + + From + + DivAssign + + Div + + Add + + Sub + + Mul { + let power_of_ten = T::e(source_scale as u32); + + values + .iter() + .map(|input| { + if input < &T::zero() { + // below 0 we ceil the number (e.g. -10.5 -> -11) + ((*input + T::one()) / power_of_ten) - T::one() + } else { + *input / power_of_ten + } + }) + .collect() +} + +fn decimal_ceil(values: &[T], source_scale: i64) -> Vec +where T: Decimal + + From + + DivAssign + + Div + + Add + + Sub + + Mul { + let power_of_ten = T::e(source_scale as u32); + + values + .iter() + .map(|input| { + if input <= &T::zero() { + *input / power_of_ten + } else { + ((*input - T::one()) / power_of_ten) + T::one() + } + }) + .collect() +} + +fn decimal_round_truncate( + arg: &ValueRef, + from_type: DataType, + dest_type: DecimalDataType, + target_scale: i64, + mode: RoundMode, +) -> Value { + let from_decimal_type = from_type.as_decimal().unwrap(); + let source_scale = from_decimal_type.scale() as i64; + + if source_scale < target_scale { + return arg.clone().to_owned(); + } + + let mut is_scalar = false; + let column = match arg { + ValueRef::Column(column) => column.clone(), + ValueRef::Scalar(s) => { + is_scalar = true; + let builder = ColumnBuilder::repeat(s, 1, &from_type); + builder.build() + } + }; + + let none_negative = target_scale >= 0; + + let result = match from_decimal_type { + DecimalDataType::Decimal128(_) => { + let (buffer, _) = i128::try_downcast_column(&column).unwrap(); + + let result = match (none_negative, mode) { + (true, RoundMode::Round) => { + decimal_round_positive::<_>(&buffer, source_scale, target_scale) + } + (true, RoundMode::Truncate) => { + decimal_truncate_positive::<_>(&buffer, source_scale, target_scale) + } + (false, RoundMode::Round) => { + decimal_round_negative::<_>(&buffer, source_scale, target_scale) + } + (false, RoundMode::Truncate) => { + decimal_truncate_negative::<_>(&buffer, source_scale, target_scale) + } + (_, RoundMode::Floor) => decimal_floor::<_>(&buffer, source_scale), + (_, RoundMode::Ceil) => decimal_ceil::<_>(&buffer, source_scale), + }; + i128::to_column(result, dest_type.size()) + } + + DecimalDataType::Decimal256(_) => { + let (buffer, _) = i256::try_downcast_column(&column).unwrap(); + let result = match (none_negative, mode) { + (true, RoundMode::Round) => { + decimal_round_positive::<_>(&buffer, source_scale, target_scale) + } + (true, RoundMode::Truncate) => { + decimal_truncate_positive::<_>(&buffer, source_scale, target_scale) + } + (false, RoundMode::Round) => { + decimal_round_negative::<_>(&buffer, source_scale, target_scale) + } + (false, RoundMode::Truncate) => { + decimal_truncate_negative::<_>(&buffer, source_scale, target_scale) + } + (_, RoundMode::Floor) => decimal_floor::<_>(&buffer, source_scale), + (_, RoundMode::Ceil) => decimal_ceil::<_>(&buffer, source_scale), + }; + i256::to_column(result, dest_type.size()) + } + }; + + let result = Column::Decimal(result); + if is_scalar { + let scalar = result.index(0).unwrap(); + Value::Scalar(scalar.to_owned()) + } else { + Value::Column(result) + } +} diff --git a/src/query/functions/src/scalars/math.rs b/src/query/functions/src/scalars/math.rs index 8197bee1cebc..a89953a91724 100644 --- a/src/query/functions/src/scalars/math.rs +++ b/src/query/functions/src/scalars/math.rs @@ -35,7 +35,11 @@ use num_traits::Float; use num_traits::Pow; use ordered_float::OrderedFloat; +use crate::scalars::decimal::register_decimal_math; + pub fn register(registry: &mut FunctionRegistry) { + register_decimal_math(registry); + registry.register_1_arg::, NumberType, _, _>( "sin", |_, _| { diff --git a/src/query/functions/tests/it/scalars/testdata/function_list.txt b/src/query/functions/tests/it/scalars/testdata/function_list.txt index ba780142d03a..8b15ea41271a 100644 --- a/src/query/functions/tests/it/scalars/testdata/function_list.txt +++ b/src/query/functions/tests/it/scalars/testdata/function_list.txt @@ -758,26 +758,27 @@ Functions overloads: 17 cbrt(Float32 NULL) :: Float64 NULL 18 cbrt(Float64) :: Float64 19 cbrt(Float64 NULL) :: Float64 NULL -0 ceil(UInt8) :: UInt8 -1 ceil(UInt8 NULL) :: UInt8 NULL -2 ceil(UInt16) :: UInt16 -3 ceil(UInt16 NULL) :: UInt16 NULL -4 ceil(UInt32) :: UInt32 -5 ceil(UInt32 NULL) :: UInt32 NULL -6 ceil(UInt64) :: UInt64 -7 ceil(UInt64 NULL) :: UInt64 NULL -8 ceil(Int8) :: Int8 -9 ceil(Int8 NULL) :: Int8 NULL -10 ceil(Int16) :: Int16 -11 ceil(Int16 NULL) :: Int16 NULL -12 ceil(Int32) :: Int32 -13 ceil(Int32 NULL) :: Int32 NULL -14 ceil(Int64) :: Int64 -15 ceil(Int64 NULL) :: Int64 NULL -16 ceil(Float32) :: Float64 -17 ceil(Float32 NULL) :: Float64 NULL -18 ceil(Float64) :: Float64 -19 ceil(Float64 NULL) :: Float64 NULL +0 ceil FACTORY +1 ceil(UInt8) :: UInt8 +2 ceil(UInt8 NULL) :: UInt8 NULL +3 ceil(UInt16) :: UInt16 +4 ceil(UInt16 NULL) :: UInt16 NULL +5 ceil(UInt32) :: UInt32 +6 ceil(UInt32 NULL) :: UInt32 NULL +7 ceil(UInt64) :: UInt64 +8 ceil(UInt64 NULL) :: UInt64 NULL +9 ceil(Int8) :: Int8 +10 ceil(Int8 NULL) :: Int8 NULL +11 ceil(Int16) :: Int16 +12 ceil(Int16 NULL) :: Int16 NULL +13 ceil(Int32) :: Int32 +14 ceil(Int32 NULL) :: Int32 NULL +15 ceil(Int64) :: Int64 +16 ceil(Int64 NULL) :: Int64 NULL +17 ceil(Float32) :: Float64 +18 ceil(Float32 NULL) :: Float64 NULL +19 ceil(Float64) :: Float64 +20 ceil(Float64 NULL) :: Float64 NULL 0 char FACTORY 1 char FACTORY 0 char_length(String) :: UInt64 @@ -1643,8 +1644,9 @@ Functions overloads: 14 factorial(Int64) :: Int64 15 factorial(Int64 NULL) :: Int64 NULL 0 flatten FACTORY -0 floor(Float64) :: Float64 -1 floor(Float64 NULL) :: Float64 NULL +0 floor FACTORY +1 floor(Float64) :: Float64 +2 floor(Float64 NULL) :: Float64 NULL 0 from_base64(String) :: String 1 from_base64(String NULL) :: String NULL 0 gen_random_uuid() :: String @@ -3009,46 +3011,47 @@ Functions overloads: 1 reverse(String NULL) :: String NULL 0 right(String, UInt64) :: String 1 right(String NULL, UInt64 NULL) :: String NULL -0 round(UInt8) :: Float64 -1 round(UInt8 NULL) :: Float64 NULL -2 round(UInt8, Int64) :: Float64 -3 round(UInt8 NULL, Int64 NULL) :: Float64 NULL -4 round(UInt16) :: Float64 -5 round(UInt16 NULL) :: Float64 NULL -6 round(UInt16, Int64) :: Float64 -7 round(UInt16 NULL, Int64 NULL) :: Float64 NULL -8 round(UInt32) :: Float64 -9 round(UInt32 NULL) :: Float64 NULL -10 round(UInt32, Int64) :: Float64 -11 round(UInt32 NULL, Int64 NULL) :: Float64 NULL -12 round(UInt64) :: Float64 -13 round(UInt64 NULL) :: Float64 NULL -14 round(UInt64, Int64) :: Float64 -15 round(UInt64 NULL, Int64 NULL) :: Float64 NULL -16 round(Int8) :: Float64 -17 round(Int8 NULL) :: Float64 NULL -18 round(Int8, Int64) :: Float64 -19 round(Int8 NULL, Int64 NULL) :: Float64 NULL -20 round(Int16) :: Float64 -21 round(Int16 NULL) :: Float64 NULL -22 round(Int16, Int64) :: Float64 -23 round(Int16 NULL, Int64 NULL) :: Float64 NULL -24 round(Int32) :: Float64 -25 round(Int32 NULL) :: Float64 NULL -26 round(Int32, Int64) :: Float64 -27 round(Int32 NULL, Int64 NULL) :: Float64 NULL -28 round(Int64) :: Float64 -29 round(Int64 NULL) :: Float64 NULL -30 round(Int64, Int64) :: Float64 -31 round(Int64 NULL, Int64 NULL) :: Float64 NULL -32 round(Float32) :: Float64 -33 round(Float32 NULL) :: Float64 NULL -34 round(Float32, Int64) :: Float64 -35 round(Float32 NULL, Int64 NULL) :: Float64 NULL -36 round(Float64) :: Float64 -37 round(Float64 NULL) :: Float64 NULL -38 round(Float64, Int64) :: Float64 -39 round(Float64 NULL, Int64 NULL) :: Float64 NULL +0 round FACTORY +1 round(UInt8) :: Float64 +2 round(UInt8 NULL) :: Float64 NULL +3 round(UInt8, Int64) :: Float64 +4 round(UInt8 NULL, Int64 NULL) :: Float64 NULL +5 round(UInt16) :: Float64 +6 round(UInt16 NULL) :: Float64 NULL +7 round(UInt16, Int64) :: Float64 +8 round(UInt16 NULL, Int64 NULL) :: Float64 NULL +9 round(UInt32) :: Float64 +10 round(UInt32 NULL) :: Float64 NULL +11 round(UInt32, Int64) :: Float64 +12 round(UInt32 NULL, Int64 NULL) :: Float64 NULL +13 round(UInt64) :: Float64 +14 round(UInt64 NULL) :: Float64 NULL +15 round(UInt64, Int64) :: Float64 +16 round(UInt64 NULL, Int64 NULL) :: Float64 NULL +17 round(Int8) :: Float64 +18 round(Int8 NULL) :: Float64 NULL +19 round(Int8, Int64) :: Float64 +20 round(Int8 NULL, Int64 NULL) :: Float64 NULL +21 round(Int16) :: Float64 +22 round(Int16 NULL) :: Float64 NULL +23 round(Int16, Int64) :: Float64 +24 round(Int16 NULL, Int64 NULL) :: Float64 NULL +25 round(Int32) :: Float64 +26 round(Int32 NULL) :: Float64 NULL +27 round(Int32, Int64) :: Float64 +28 round(Int32 NULL, Int64 NULL) :: Float64 NULL +29 round(Int64) :: Float64 +30 round(Int64 NULL) :: Float64 NULL +31 round(Int64, Int64) :: Float64 +32 round(Int64 NULL, Int64 NULL) :: Float64 NULL +33 round(Float32) :: Float64 +34 round(Float32 NULL) :: Float64 NULL +35 round(Float32, Int64) :: Float64 +36 round(Float32 NULL, Int64 NULL) :: Float64 NULL +37 round(Float64) :: Float64 +38 round(Float64 NULL) :: Float64 NULL +39 round(Float64, Int64) :: Float64 +40 round(Float64 NULL, Int64 NULL) :: Float64 NULL 0 rpad(String, UInt64, String) :: String 1 rpad(String NULL, UInt64 NULL, String NULL) :: String NULL 0 rtrim(String) :: String @@ -3647,46 +3650,47 @@ Functions overloads: 1 trim_leading(String NULL, String NULL) :: String NULL 0 trim_trailing(String, String) :: String 1 trim_trailing(String NULL, String NULL) :: String NULL -0 truncate(UInt8) :: Float64 -1 truncate(UInt8 NULL) :: Float64 NULL -2 truncate(UInt8, Int64) :: Float64 -3 truncate(UInt8 NULL, Int64 NULL) :: Float64 NULL -4 truncate(UInt16) :: Float64 -5 truncate(UInt16 NULL) :: Float64 NULL -6 truncate(UInt16, Int64) :: Float64 -7 truncate(UInt16 NULL, Int64 NULL) :: Float64 NULL -8 truncate(UInt32) :: Float64 -9 truncate(UInt32 NULL) :: Float64 NULL -10 truncate(UInt32, Int64) :: Float64 -11 truncate(UInt32 NULL, Int64 NULL) :: Float64 NULL -12 truncate(UInt64) :: Float64 -13 truncate(UInt64 NULL) :: Float64 NULL -14 truncate(UInt64, Int64) :: Float64 -15 truncate(UInt64 NULL, Int64 NULL) :: Float64 NULL -16 truncate(Int8) :: Float64 -17 truncate(Int8 NULL) :: Float64 NULL -18 truncate(Int8, Int64) :: Float64 -19 truncate(Int8 NULL, Int64 NULL) :: Float64 NULL -20 truncate(Int16) :: Float64 -21 truncate(Int16 NULL) :: Float64 NULL -22 truncate(Int16, Int64) :: Float64 -23 truncate(Int16 NULL, Int64 NULL) :: Float64 NULL -24 truncate(Int32) :: Float64 -25 truncate(Int32 NULL) :: Float64 NULL -26 truncate(Int32, Int64) :: Float64 -27 truncate(Int32 NULL, Int64 NULL) :: Float64 NULL -28 truncate(Int64) :: Float64 -29 truncate(Int64 NULL) :: Float64 NULL -30 truncate(Int64, Int64) :: Float64 -31 truncate(Int64 NULL, Int64 NULL) :: Float64 NULL -32 truncate(Float32) :: Float64 -33 truncate(Float32 NULL) :: Float64 NULL -34 truncate(Float32, Int64) :: Float64 -35 truncate(Float32 NULL, Int64 NULL) :: Float64 NULL -36 truncate(Float64) :: Float64 -37 truncate(Float64 NULL) :: Float64 NULL -38 truncate(Float64, Int64) :: Float64 -39 truncate(Float64 NULL, Int64 NULL) :: Float64 NULL +0 truncate FACTORY +1 truncate(UInt8) :: Float64 +2 truncate(UInt8 NULL) :: Float64 NULL +3 truncate(UInt8, Int64) :: Float64 +4 truncate(UInt8 NULL, Int64 NULL) :: Float64 NULL +5 truncate(UInt16) :: Float64 +6 truncate(UInt16 NULL) :: Float64 NULL +7 truncate(UInt16, Int64) :: Float64 +8 truncate(UInt16 NULL, Int64 NULL) :: Float64 NULL +9 truncate(UInt32) :: Float64 +10 truncate(UInt32 NULL) :: Float64 NULL +11 truncate(UInt32, Int64) :: Float64 +12 truncate(UInt32 NULL, Int64 NULL) :: Float64 NULL +13 truncate(UInt64) :: Float64 +14 truncate(UInt64 NULL) :: Float64 NULL +15 truncate(UInt64, Int64) :: Float64 +16 truncate(UInt64 NULL, Int64 NULL) :: Float64 NULL +17 truncate(Int8) :: Float64 +18 truncate(Int8 NULL) :: Float64 NULL +19 truncate(Int8, Int64) :: Float64 +20 truncate(Int8 NULL, Int64 NULL) :: Float64 NULL +21 truncate(Int16) :: Float64 +22 truncate(Int16 NULL) :: Float64 NULL +23 truncate(Int16, Int64) :: Float64 +24 truncate(Int16 NULL, Int64 NULL) :: Float64 NULL +25 truncate(Int32) :: Float64 +26 truncate(Int32 NULL) :: Float64 NULL +27 truncate(Int32, Int64) :: Float64 +28 truncate(Int32 NULL, Int64 NULL) :: Float64 NULL +29 truncate(Int64) :: Float64 +30 truncate(Int64 NULL) :: Float64 NULL +31 truncate(Int64, Int64) :: Float64 +32 truncate(Int64 NULL, Int64 NULL) :: Float64 NULL +33 truncate(Float32) :: Float64 +34 truncate(Float32 NULL) :: Float64 NULL +35 truncate(Float32, Int64) :: Float64 +36 truncate(Float32 NULL, Int64 NULL) :: Float64 NULL +37 truncate(Float64) :: Float64 +38 truncate(Float64 NULL) :: Float64 NULL +39 truncate(Float64, Int64) :: Float64 +40 truncate(Float64 NULL, Int64 NULL) :: Float64 NULL 0 try_inet_aton(String) :: UInt32 NULL 1 try_inet_aton(String NULL) :: UInt32 NULL 0 try_inet_ntoa(Int64) :: String NULL diff --git a/src/query/functions/tests/it/scalars/testdata/math.txt b/src/query/functions/tests/it/scalars/testdata/math.txt index 4f3f372b222c..d959a713f93c 100644 --- a/src/query/functions/tests/it/scalars/testdata/math.txt +++ b/src/query/functions/tests/it/scalars/testdata/math.txt @@ -202,9 +202,9 @@ output : 5 ast : ceil(5.6) raw expr : ceil(5.6) -checked expr : ceil(to_float32(5.6_d128(2,1))) -optimized expr : 6_f64 -output type : Float64 +checked expr : ceil(5.6_d128(2,1)) +optimized expr : 6_d128(2,0) +output type : Decimal(2, 0) output domain : {6..=6} output : 6 @@ -281,47 +281,47 @@ evaluation (internal): ast : round(-1.23) raw expr : round(minus(1.23)) -checked expr : round(to_float32(minus(1.23_d128(3,2)))) -optimized expr : -1_f64 -output type : Float64 +checked expr : round(76)(minus(1.23_d128(3,2))) +optimized expr : -1_d128(3,0) +output type : Decimal(3, 0) output domain : {-1..=-1} output : -1 ast : round(1.298, 1) raw expr : round(1.298, 1) -checked expr : round(to_float32(1.298_d128(4,3)), to_int64(1_u8)) -optimized expr : 1.3_f64 -output type : Float64 +checked expr : round(77)(1.298_d128(4,3), 1_u8) +optimized expr : 1.3_d128(4,1) +output type : Decimal(4, 1) output domain : {1.3..=1.3} output : 1.3 ast : round(1.298, 0) raw expr : round(1.298, 0) -checked expr : round(to_float32(1.298_d128(4,3)), to_int64(0_u8)) -optimized expr : 1_f64 -output type : Float64 +checked expr : round(76)(1.298_d128(4,3), 0_u8) +optimized expr : 1_d128(4,0) +output type : Decimal(4, 0) output domain : {1..=1} output : 1 ast : round(23.298, -1) raw expr : round(23.298, minus(1)) -checked expr : round(to_float32(23.298_d128(5,3)), to_int64(minus(1_u8))) -optimized expr : 20_f64 -output type : Float64 +checked expr : round(75)(23.298_d128(5,3), minus(1_u8)) +optimized expr : 20_d128(5,0) +output type : Decimal(5, 0) output domain : {20..=20} output : 20 ast : round(0.12345678901234567890123456789012345, 35) raw expr : round(0.12345678901234567890123456789012345, 35) -checked expr : round(to_float32(0.12345678901234567890123456789012345_d128(35,35)), to_int64(35_u8)) -optimized expr : 0.1234567835_f64 -output type : Float64 -output domain : {0.1234567835..=0.1234567835} -output : 0.1234567835 +checked expr : round(111)(0.12345678901234567890123456789012345_d128(35,35), 35_u8) +optimized expr : 0.12345678901234567890123456789012345_d128(35,35) +output type : Decimal(35, 35) +output domain : {0.12345678901234567890123456789012345..=0.12345678901234567890123456789012345} +output : 0.12345678901234567890123456789012345 ast : round(a) @@ -410,27 +410,27 @@ evaluation (internal): ast : truncate(1.223, 1) raw expr : truncate(1.223, 1) -checked expr : truncate(to_float32(1.223_d128(4,3)), to_int64(1_u8)) -optimized expr : 1.2_f64 -output type : Float64 +checked expr : truncate(77)(1.223_d128(4,3), 1_u8) +optimized expr : 1.2_d128(4,1) +output type : Decimal(4, 1) output domain : {1.2..=1.2} output : 1.2 ast : truncate(1.999) raw expr : truncate(1.999) -checked expr : truncate(to_float32(1.999_d128(4,3))) -optimized expr : 1_f64 -output type : Float64 +checked expr : truncate(76)(1.999_d128(4,3)) +optimized expr : 1_d128(4,0) +output type : Decimal(4, 0) output domain : {1..=1} output : 1 ast : truncate(1.999, 1) raw expr : truncate(1.999, 1) -checked expr : truncate(to_float32(1.999_d128(4,3)), to_int64(1_u8)) -optimized expr : 1.9_f64 -output type : Float64 +checked expr : truncate(77)(1.999_d128(4,3), 1_u8) +optimized expr : 1.9_d128(4,1) +output type : Decimal(4, 1) output domain : {1.9..=1.9} output : 1.9 @@ -446,9 +446,9 @@ output : 100 ast : truncate(10.28*100, 0) raw expr : truncate(multiply(10.28, 100), 0) -checked expr : truncate(to_float32(multiply(to_decimal(7, 2)(10.28_d128(4,2)), to_decimal(7, 0)(100_u8))), to_int64(0_u8)) -optimized expr : 1028_f64 -output type : Float64 +checked expr : truncate(76)(multiply(to_decimal(7, 2)(10.28_d128(4,2)), to_decimal(7, 0)(100_u8)), 0_u8) +optimized expr : 1028_d128(7,0) +output type : Decimal(7, 0) output domain : {1028..=1028} output : 1028 @@ -521,26 +521,12 @@ output domain : {0.6931471805..=0.6931471805} output : 0.6931471805 -ast : round(2, a) -raw expr : round(2, a::Int64) -checked expr : round(2_u8, a) -evaluation: -+--------+--------------+--------------+ -| | a | Output | -+--------+--------------+--------------+ -| Type | Int64 | Float64 | -| Domain | {10..=65536} | {-inf..=NaN} | -| Row 0 | 22 | 2 | -| Row 1 | 65536 | 2 | -| Row 2 | 10 | 2 | -+--------+--------------+--------------+ -evaluation (internal): -+--------+------------------------+ -| Column | Data | -+--------+------------------------+ -| a | Int64([22, 65536, 10]) | -| Output | Float64([2, 2, 2]) | -+--------+------------------------+ +error: + --> SQL:1:10 + | +1 | round(2, a) + | ^ Need constant number. + ast : factorial(5) diff --git a/src/tests/sqlsmith/src/sql_gen/func.rs b/src/tests/sqlsmith/src/sql_gen/func.rs index 9e867e86d8c3..90e96e40e6ed 100644 --- a/src/tests/sqlsmith/src/sql_gen/func.rs +++ b/src/tests/sqlsmith/src/sql_gen/func.rs @@ -155,8 +155,8 @@ impl<'a, R: Rng> SqlGenerator<'a, R> { 0 => "and_filters".to_string(), 1 => "regexp_like".to_string(), 2 => { - let comp_func = ["eq", "gt", "gte", "lt", "lte", "ne", "noteq"]; - comp_func[self.rng.gen_range(0..=6)].to_string() + let comp_func = ["eq", "gt", "gte", "lt", "lte", "noteq"]; + comp_func[self.rng.gen_range(0..=5)].to_string() } 3 => "ignore".to_string(), diff --git a/tests/sqllogictests/suites/query/02_function/02_0014_function_maths.test b/tests/sqllogictests/suites/query/02_function/02_0014_function_maths.test index d2780f51f7fd..c8c5c88bbc3a 100644 --- a/tests/sqllogictests/suites/query/02_function/02_0014_function_maths.test +++ b/tests/sqllogictests/suites/query/02_function/02_0014_function_maths.test @@ -342,3 +342,25 @@ query I select abs(-9223372036854775808) ---- 9223372036854775808 + +query IIIIII +SELECT floor(123.45), floor(-123.45), floor(0.0), floor(123456789.123), floor(-123456789.123), floor(123456.7891) +---- +123 -124 0 123456789 -123456790 123456 + +query IIIIII +SELECT ceil(123.45), ceil(-123.45), ceil(0.0), ceil(123456789.123), ceil(-123456789.123), ceil(123456.7891) +---- +124 -123 0 123456790 -123456789 123457 + + +query FFFFFFFFFF +SELECT truncate(123.4567, 2), truncate(-123.4567, 2), truncate(123.4567, 0), truncate(123.456789123, 6), truncate(0.0, 2), truncate(123456789.123, 2), truncate(-123456789.123, 2), truncate(123456.7891, 1), truncate(123456789012345.12345, 3), truncate(123.45, -1) +---- +123.45 -123.45 123 123.456789 0.0 123456789.12 -123456789.12 123456.7 123456789012345.123 120 + + +query FFFFFFFFFF +SELECT round(123.4567, 2), round(-123.4567, 2), round(123.455, 2), round(123.445, 2), round(123.456789123, 6), round(0.0, 2), round(123456789.123, 2), round(-123456789.123, 2), round(123456.7891, 1), round(123456.789, -2), round(-123456.789, -3), round(123456789012345.12345, 3) +---- +123.46 -123.46 123.46 123.45 123.456789 0.0 123456789.12 -123456789.12 123456.8 123500 -123000 123456789012345.123 diff --git a/tests/sqllogictests/suites/tpcds/queries.test b/tests/sqllogictests/suites/tpcds/queries.test index 3582c9dceeb2..042890ab6a4d 100644 --- a/tests/sqllogictests/suites/tpcds/queries.test +++ b/tests/sqllogictests/suites/tpcds/queries.test @@ -215,62 +215,62 @@ FROM WHERE d_week_seq1 = d_week_seq2-53 ORDER BY d_week_seq1 NULLS FIRST; ---- -5270 NULL 2.6 NULL 0.71 2.48 NULL NULL -5270 NULL 2.6 NULL 0.71 2.48 NULL NULL -5270 NULL 2.6 NULL 0.71 2.48 NULL NULL -5270 NULL 2.6 NULL 0.71 2.48 NULL NULL -5270 NULL 2.6 NULL 0.71 2.48 NULL NULL -5270 NULL 2.6 NULL 0.71 2.48 NULL NULL -5270 NULL 2.6 NULL 0.71 2.48 NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL -5271 NULL NULL 0.5 NULL NULL NULL NULL +5270 NULL 2.60 NULL 0.71 2.48 NULL NULL +5270 NULL 2.60 NULL 0.71 2.48 NULL NULL +5270 NULL 2.60 NULL 0.71 2.48 NULL NULL +5270 NULL 2.60 NULL 0.71 2.48 NULL NULL +5270 NULL 2.60 NULL 0.71 2.48 NULL NULL +5270 NULL 2.60 NULL 0.71 2.48 NULL NULL +5270 NULL 2.60 NULL 0.71 2.48 NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL +5271 NULL NULL 0.50 NULL NULL NULL NULL 5272 NULL NULL NULL 0.52 1.42 NULL 0.49 5272 NULL NULL NULL 0.52 1.42 NULL 0.49 5272 NULL NULL NULL 0.52 1.42 NULL 0.49 @@ -761,55 +761,55 @@ ORDER BY d_week_seq1 NULLS FIRST; 5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL -5285 0.8 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL +5285 0.80 0.71 NULL NULL NULL NULL NULL 5286 1.62 NULL NULL NULL NULL NULL NULL 5286 1.62 NULL NULL NULL NULL NULL NULL 5286 1.62 NULL NULL NULL NULL NULL NULL @@ -859,55 +859,55 @@ ORDER BY d_week_seq1 NULLS FIRST; 5286 1.62 NULL NULL NULL NULL NULL NULL 5286 1.62 NULL NULL NULL NULL NULL NULL 5286 1.62 NULL NULL NULL NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL -5287 NULL NULL NULL 1.9 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL +5287 NULL NULL NULL 1.90 NULL NULL NULL 5288 1.17 NULL NULL NULL NULL NULL NULL 5288 1.17 NULL NULL NULL NULL NULL NULL 5288 1.17 NULL NULL NULL NULL NULL NULL @@ -1251,55 +1251,55 @@ ORDER BY d_week_seq1 NULLS FIRST; 5300 NULL 0.93 NULL NULL NULL 2.01 NULL 5300 NULL 0.93 NULL NULL NULL 2.01 NULL 5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.7 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 +5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 5302 NULL NULL NULL 0.18 NULL NULL NULL 5302 NULL NULL NULL 0.18 NULL NULL NULL 5302 NULL NULL NULL 0.18 NULL NULL NULL @@ -1398,55 +1398,55 @@ ORDER BY d_week_seq1 NULLS FIRST; 5304 NULL NULL 3.64 NULL NULL NULL NULL 5304 NULL NULL 3.64 NULL NULL NULL NULL 5304 NULL NULL 3.64 NULL NULL NULL NULL -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.2 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 +5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 5306 NULL 1.92 NULL NULL NULL NULL NULL 5306 NULL 1.92 NULL NULL NULL NULL NULL 5306 NULL 1.92 NULL NULL NULL NULL NULL @@ -2035,55 +2035,55 @@ ORDER BY d_week_seq1 NULLS FIRST; 5318 NULL NULL NULL NULL NULL NULL 0.29 5318 NULL NULL NULL NULL NULL NULL 0.29 5318 NULL NULL NULL NULL NULL NULL 0.29 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 -5319 NULL NULL NULL NULL NULL NULL 0.8 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 +5319 NULL NULL NULL NULL NULL NULL 0.80 5320 NULL NULL NULL NULL NULL 1.87 NULL 5320 NULL NULL NULL NULL NULL 1.87 NULL 5320 NULL NULL NULL NULL NULL 1.87 NULL @@ -8493,7 +8493,7 @@ LIMIT 100; 2000 8 501 1.69 71 88.22 88.76 42 59.15 33.59 2000 8 568 5.69 91 106.86 72.31 16 88.70 22.13 2000 8 607 1.24 94 69.87 8.44 76 15.60 12.40 -2000 8 937 4.0 32 28.49 43.91 8 35.73 79.14 +2000 8 937 4.00 32 28.49 43.91 8 35.73 79.14 2000 11 99 3.17 73 35.63 42.50 23 82.52 81.30 2000 11 344 0.98 79 55.29 9.50 81 60.39 6.55 2000 11 937 1.46 95 65.04 109.88 65 69.73 181.29 @@ -8505,20 +8505,20 @@ LIMIT 100; 2000 19 539 0.49 33 32.33 41.35 68 124.61 67.61 2000 19 566 0.61 53 80.97 85.69 87 80.36 84.40 2000 19 682 0.48 47 58.06 39.08 97 13.10 18.56 -2000 19 922 26.5 53 32.46 10.43 2 36.78 51.51 +2000 19 922 26.50 53 32.46 10.43 2 36.78 51.51 2000 20 321 2.38 31 17.31 17.66 13 84.13 160.90 -2000 20 501 0.3 10 4.27 1.33 33 12.91 0.30 +2000 20 501 0.30 10 4.27 1.33 33 12.91 0.30 2000 20 822 1.17 69 86.20 136.53 59 61.54 130.02 2000 25 665 0.11 7 31.24 17.68 65 85.15 143.04 2000 26 354 0.19 18 5.00 4.84 96 89.99 166.35 -2000 26 498 0.4 24 57.11 26.55 60 85.59 76.73 +2000 26 498 0.40 24 57.11 26.55 60 85.59 76.73 2000 26 784 0.95 69 7.68 5.25 73 76.72 121.29 2000 26 878 2.07 58 7.20 4.41 28 53.28 28.79 -2000 29 65 16.4 82 18.74 2.16 5 81.49 12.09 +2000 29 65 16.40 82 18.74 2.16 5 81.49 12.09 2000 29 335 0.35 32 76.21 8.09 91 2.15 1.16 2000 29 501 1.76 86 37.64 18.80 49 95.25 65.69 2000 29 843 1.04 73 59.61 61.15 70 23.06 20.37 -2000 29 990 0.2 10 62.00 53.94 49 88.35 21.73 +2000 29 990 0.20 10 62.00 53.94 49 88.35 21.73 2000 29 994 0.53 16 94.38 60.13 30 1.47 1.36 2000 31 4 0.93 13 87.20 36.16 14 81.43 75.69 2000 31 182 0.38 29 66.79 45.75 77 51.10 66.77 @@ -8544,7 +8544,7 @@ LIMIT 100; 2000 49 445 1.17 91 79.33 2.74 78 49.74 9.40 2000 50 193 2.41 94 63.22 32.52 39 1.80 1.71 2000 50 194 0.23 22 97.27 177.37 94 25.46 54.04 -2000 50 391 0.2 16 59.10 37.51 81 96.87 94.94 +2000 50 391 0.20 16 59.10 37.51 81 96.87 94.94 2000 50 393 1.05 67 82.86 21.75 64 82.54 154.31 2000 50 962 2.63 84 4.69 1.01 32 62.01 101.08 2000 50 985 0.26 24 17.04 8.34 93 20.29 33.41 @@ -8563,23 +8563,23 @@ LIMIT 100; 2000 61 602 0.13 7 57.58 54.01 52 65.10 28.35 2000 61 684 0.53 51 5.35 1.50 97 63.81 16.25 2000 61 982 0.09 7 26.83 25.22 74 43.88 69.07 -2000 62 498 0.3 27 55.17 36.57 89 2.42 2.17 -2000 65 169 9.4 47 93.06 109.67 5 36.69 51.72 +2000 62 498 0.30 27 55.17 36.57 89 2.42 2.17 +2000 65 169 9.40 47 93.06 109.67 5 36.69 51.72 2000 65 182 4.67 56 19.06 12.63 12 98.17 137.55 2000 65 928 0.48 25 13.75 5.51 52 57.55 32.87 2000 65 933 0.86 79 86.28 132.10 92 94.69 187.89 -2000 67 94 2.5 35 28.17 13.23 14 39.29 37.31 -2000 67 761 12.2 61 20.72 9.13 5 27.61 25.35 +2000 67 94 2.50 35 28.17 13.23 14 39.29 37.31 +2000 67 761 12.20 61 20.72 9.13 5 27.61 25.35 2000 67 841 1.01 79 41.08 29.17 78 25.77 34.78 2000 67 878 0.83 76 22.82 14.77 92 82.50 100.75 -2000 68 37 0.1 9 94.94 67.67 91 74.13 130.52 +2000 68 37 0.10 9 94.94 67.67 91 74.13 130.52 2000 68 65 2.76 69 15.89 4.09 25 25.80 22.95 2000 68 122 1.95 76 28.79 1.18 39 17.33 22.92 2000 68 375 2.88 95 30.91 20.69 33 84.01 46.92 2000 68 391 5.64 62 53.06 37.93 11 25.78 37.76 2000 68 808 2.28 41 25.44 0.45 18 89.72 117.53 2000 71 668 0.89 65 43.58 8.59 73 68.38 9.23 -2000 73 596 3.7 74 69.38 6.32 20 88.71 1.33 +2000 73 596 3.70 74 69.38 6.32 20 88.71 1.33 # Q79 onlyif mysql diff --git a/tests/sqllogictests/suites/tpch/queries.test b/tests/sqllogictests/suites/tpch/queries.test index df4fe3cadce9..fa4dc273e1a3 100644 --- a/tests/sqllogictests/suites/tpch/queries.test +++ b/tests/sqllogictests/suites/tpch/queries.test @@ -237,11 +237,11 @@ group by order by revenue desc; ---- -CHINA 782211.0 -INDIA 637613.0 -JAPAN 600008.0 -INDONESIA 558048.0 -VIETNAM 449785.0 +CHINA 782211 +INDIA 637613 +JAPAN 600008 +INDONESIA 558048 +VIETNAM 449785 # Q6 query I @@ -255,7 +255,7 @@ where and l_discount between 0.05 and 0.07 and l_quantity < 24; ---- -11803420.0 +11803420.253 # Q7 query I @@ -299,10 +299,10 @@ order by cust_nation, l_year; ---- -FRANCE GERMANY 1995 4637235.0 -FRANCE GERMANY 1996 5224780.0 -GERMANY FRANCE 1995 6232819.0 -GERMANY FRANCE 1996 5557312.5 +FRANCE GERMANY 1995 4637235.150 +FRANCE GERMANY 1996 5224779.573 +GERMANY FRANCE 1995 6232818.703 +GERMANY FRANCE 1996 5557312.112 # Q8 query I @@ -382,11 +382,11 @@ order by sum_profit limit 5; ---- -MOZAMBIQUE 1998 162042.0 -JORDAN 1998 181148.0 -MOROCCO 1998 181533.0 -JAPAN 1998 184953.0 -VIETNAM 1998 192431.0 +MOZAMBIQUE 1998 162042 +JORDAN 1998 181148 +MOROCCO 1998 181533 +JAPAN 1998 184953 +VIETNAM 1998 192431 # Q10 query I @@ -422,11 +422,11 @@ group by order by revenue desc limit 5; ---- -8242 Customer#000008242 622786.687 6322.09 ETHIOPIA P2n4nJhy,UqSo2s43YfSvYJDZ6lk 15-792-676-1184 slyly regular packages haggle carefully ironic ideas. courts are furiously. furiously unusual theodolites cajole. i -7714 Customer#000007714 557400.312 9799.98 IRAN SnnIGB,SkmnWpX3 20-922-418-6024 arhorses according to the blithely express re -11032 Customer#000011032 512500.937 8496.93 UNITED KINGDOM WIKHC7K3Cn7156iNOyfVG3cZ7YqkgsR,Ly 33-102-772-3533 posits-- furiously ironic accounts are again -2455 Customer#000002455 481592.437 2070.99 GERMANY RVn1ZSRtLqPlJLIZxvpmsbgC02 17-946-225-9977 al asymptotes. finally ironic accounts cajole furiously. permanently unusual theodolites aro -12106 Customer#000012106 479414.218 5342.11 UNITED STATES wth3twOmu6vy 34-905-346-4472 ly after the blithely regular foxes. accounts haggle carefully alongside of the blithely even ideas. +8242 Customer#000008242 622786.729 6322.09 ETHIOPIA P2n4nJhy,UqSo2s43YfSvYJDZ6lk 15-792-676-1184 slyly regular packages haggle carefully ironic ideas. courts are furiously. furiously unusual theodolites cajole. i +7714 Customer#000007714 557400.305 9799.98 IRAN SnnIGB,SkmnWpX3 20-922-418-6024 arhorses according to the blithely express re +11032 Customer#000011032 512500.964 8496.93 UNITED KINGDOM WIKHC7K3Cn7156iNOyfVG3cZ7YqkgsR,Ly 33-102-772-3533 posits-- furiously ironic accounts are again +2455 Customer#000002455 481592.405 2070.99 GERMANY RVn1ZSRtLqPlJLIZxvpmsbgC02 17-946-225-9977 al asymptotes. finally ironic accounts cajole furiously. permanently unusual theodolites aro +12106 Customer#000012106 479414.213 5342.11 UNITED STATES wth3twOmu6vy 34-905-346-4472 ly after the blithely regular foxes. accounts haggle carefully alongside of the blithely even ideas. # Q11 query I @@ -708,7 +708,7 @@ where order by s_suppkey; ---- -677 Supplier#000000677 8mhrffG7D2WJBSQbOGstQ 23-290-639-3315 1614410.37 +677 Supplier#000000677 8mhrffG7D2WJBSQbOGstQ 23-290-639-3315 1614410.29 # Q15 with materialized cte query I @@ -743,7 +743,7 @@ where order by s_suppkey; ---- -677 Supplier#000000677 8mhrffG7D2WJBSQbOGstQ 23-290-639-3315 1614410.37 +677 Supplier#000000677 8mhrffG7D2WJBSQbOGstQ 23-290-639-3315 1614410.29 # Q16 @@ -821,7 +821,7 @@ where l_partkey = p_partkey ); ---- -23512.75195312 +23512.75285714 #Q17 variant query I @@ -843,7 +843,7 @@ WHERE l_partkey = p_partkey ) ---- -23512.75195312 +23512.75285714 #Q18 query I @@ -958,7 +958,7 @@ where ) ; ---- -350370.468 +350370.483 # Q20 query I From 45ca98549f0788acad4711eabf0d187a58cfd9ca Mon Sep 17 00:00:00 2001 From: RinChanNOW Date: Mon, 18 Dec 2023 10:12:18 +0800 Subject: [PATCH 02/20] refactor: abstract a common merger to do merge sort. (#14020) * refactor: abstract a common merger to do merge sort. * Replace the merger in `TransformSortSpill` with `HeapMerger`. * Replace the merger in `TransformSortMerge` with `HeapMerger`. * Ensure merger has at least two input streams. * Refactor `SortedStream` trait, output block along with order column. * Add `limit` to `HeapMerger`. * Refactor `HeapMerger`. * Replace the merger in `TransformMultiSortMerge` with `HeapMerger`. * Fix and add assertions. * Improve. * Recover `TransformMultiSortMerge`. * Refactor codes. --- Cargo.lock | 2 + src/query/expression/src/block.rs | 7 + src/query/pipeline/transforms/Cargo.toml | 4 + src/query/pipeline/transforms/src/lib.rs | 1 + .../src/processors/transforms/sort/cursor.rs | 7 + .../src/processors/transforms/sort/merger.rs | 321 ++++++++++++++++++ .../src/processors/transforms/sort/mod.rs | 2 + .../processors/transforms/sort/rows/common.rs | 6 +- .../processors/transforms/sort/rows/mod.rs | 16 +- .../processors/transforms/sort/rows/simple.rs | 11 +- .../src/processors/transforms/sort/utils.rs | 38 +++ .../transforms/transform_multi_sort_merge.rs | 12 +- .../transforms/transform_sort_merge.rs | 154 ++++----- .../transforms/transform_sort_merge_base.rs | 57 ++-- .../pipeline/transforms/tests/it/main.rs | 15 + .../pipeline/transforms/tests/it/merger.rs | 262 ++++++++++++++ .../src/pipelines/builders/builder_sort.rs | 55 +-- .../transforms/transform_sort_spill.rs | 174 ++-------- .../executor/physical_plans/physical_sort.rs | 5 +- 19 files changed, 825 insertions(+), 324 deletions(-) create mode 100644 src/query/pipeline/transforms/src/processors/transforms/sort/merger.rs create mode 100644 src/query/pipeline/transforms/tests/it/main.rs create mode 100644 src/query/pipeline/transforms/tests/it/merger.rs diff --git a/Cargo.lock b/Cargo.lock index bc40520848dc..e5ac5940a8cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3233,8 +3233,10 @@ dependencies = [ "databend-common-expression", "databend-common-pipeline-core", "databend-common-profile", + "itertools 0.10.5", "jsonb 0.3.0 (git+https://github.com/datafuselabs/jsonb?rev=582c139)", "match-template", + "rand 0.8.5", "serde", "typetag", ] diff --git a/src/query/expression/src/block.rs b/src/query/expression/src/block.rs index 1e46884df4bd..029239fc5c70 100644 --- a/src/query/expression/src/block.rs +++ b/src/query/expression/src/block.rs @@ -553,6 +553,13 @@ impl DataBlock { } DataBlock::new_with_meta(columns, self.num_rows, self.meta) } + + #[inline] + pub fn get_last_column(&self) -> &Column { + debug_assert!(!self.columns.is_empty()); + debug_assert!(self.columns.last().unwrap().value.as_column().is_some()); + self.columns.last().unwrap().value.as_column().unwrap() + } } impl TryFrom for ArrowChunk { diff --git a/src/query/pipeline/transforms/Cargo.toml b/src/query/pipeline/transforms/Cargo.toml index 0630c15e4654..3fff363caa10 100644 --- a/src/query/pipeline/transforms/Cargo.toml +++ b/src/query/pipeline/transforms/Cargo.toml @@ -22,5 +22,9 @@ match-template = { workspace = true } serde = { workspace = true } typetag = { workspace = true } +[dev-dependencies] +itertools = { workspace = true } +rand = { workspace = true } + [package.metadata.cargo-machete] ignored = ["match-template"] diff --git a/src/query/pipeline/transforms/src/lib.rs b/src/query/pipeline/transforms/src/lib.rs index addeeb815e23..339e62d63f84 100644 --- a/src/query/pipeline/transforms/src/lib.rs +++ b/src/query/pipeline/transforms/src/lib.rs @@ -15,5 +15,6 @@ #![feature(core_intrinsics)] #![feature(int_roundings)] #![feature(binary_heap_as_slice)] +#![feature(let_chains)] pub mod processors; diff --git a/src/query/pipeline/transforms/src/processors/transforms/sort/cursor.rs b/src/query/pipeline/transforms/src/processors/transforms/sort/cursor.rs index 85804e80363a..4c6c422d77ea 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sort/cursor.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sort/cursor.rs @@ -14,6 +14,8 @@ use std::cmp::Ordering; +use databend_common_expression::Column; + use super::rows::Rows; /// A cursor point to a certain row in a data block. @@ -64,6 +66,11 @@ impl Cursor { pub fn num_rows(&self) -> usize { self.num_rows } + + #[inline] + pub fn to_column(&self) -> Column { + self.rows.to_column() + } } impl Ord for Cursor { diff --git a/src/query/pipeline/transforms/src/processors/transforms/sort/merger.rs b/src/query/pipeline/transforms/src/processors/transforms/sort/merger.rs new file mode 100644 index 000000000000..a2ae12462731 --- /dev/null +++ b/src/query/pipeline/transforms/src/processors/transforms/sort/merger.rs @@ -0,0 +1,321 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::cmp::Reverse; +use std::collections::BinaryHeap; +use std::collections::VecDeque; +use std::sync::Arc; + +use databend_common_exception::Result; +use databend_common_expression::Column; +use databend_common_expression::DataBlock; +use databend_common_expression::DataSchemaRef; +use databend_common_expression::SortColumnDescription; + +use super::utils::find_bigger_child_of_root; +use super::Cursor; +use super::Rows; + +#[async_trait::async_trait] +pub trait SortedStream { + /// Returns the next block with the order column and if it is pending. + /// + /// If the block is [None] and it's not pending, it means the stream is finished. + /// If the block is [None] but it's pending, it means the stream is not finished yet. + fn next(&mut self) -> Result<(Option<(DataBlock, Column)>, bool)> { + Ok((None, false)) + } + + /// The async version of `next`. + async fn async_next(&mut self) -> Result<(Option<(DataBlock, Column)>, bool)> { + self.next() + } +} + +/// A merge sort operator to merge multiple sorted streams and output one sorted stream. +pub struct HeapMerger +where + R: Rows, + S: SortedStream, +{ + schema: DataSchemaRef, + sort_desc: Arc>, + unsorted_streams: Vec, + heap: BinaryHeap>>, + buffer: Vec, + pending_streams: VecDeque, + batch_size: usize, + limit: Option, + + temp_sorted_num_rows: usize, + temp_output_indices: Vec<(usize, usize, usize)>, + temp_sorted_blocks: Vec, +} + +impl HeapMerger +where + R: Rows, + S: SortedStream + Send, +{ + pub fn create( + schema: DataSchemaRef, + streams: Vec, + sort_desc: Arc>, + batch_size: usize, + limit: Option, + ) -> Self { + // We only create a merger when there are at least two streams. + debug_assert!(streams.len() > 1, "streams.len() = {}", streams.len()); + + let heap = BinaryHeap::with_capacity(streams.len()); + let buffer = vec![DataBlock::empty_with_schema(schema.clone()); streams.len()]; + let pending_stream = (0..streams.len()).collect(); + + Self { + schema, + unsorted_streams: streams, + heap, + buffer, + batch_size, + limit, + sort_desc, + pending_streams: pending_stream, + temp_sorted_num_rows: 0, + temp_output_indices: vec![], + temp_sorted_blocks: vec![], + } + } + + #[inline(always)] + pub fn is_finished(&self) -> bool { + (self.heap.is_empty() && !self.has_pending_stream() && self.temp_sorted_num_rows == 0) + || self.limit == Some(0) + } + + #[inline(always)] + pub fn has_pending_stream(&self) -> bool { + !self.pending_streams.is_empty() + } + + // This method can only be called when there is no data of the stream in the heap. + pub async fn async_poll_pending_stream(&mut self) -> Result<()> { + let mut continue_pendings = Vec::new(); + while let Some(i) = self.pending_streams.pop_front() { + debug_assert!(self.buffer[i].is_empty()); + let (input, pending) = self.unsorted_streams[i].async_next().await?; + if pending { + continue_pendings.push(i); + continue; + } + if let Some((block, col)) = input { + let rows = R::from_column(&col, &self.sort_desc)?; + let cursor = Cursor::new(i, rows); + self.heap.push(Reverse(cursor)); + self.buffer[i] = block; + } + } + self.pending_streams.extend(continue_pendings); + Ok(()) + } + + #[inline] + pub fn poll_pending_stream(&mut self) -> Result<()> { + let mut continue_pendings = Vec::new(); + while let Some(i) = self.pending_streams.pop_front() { + debug_assert!(self.buffer[i].is_empty()); + let (input, pending) = self.unsorted_streams[i].next()?; + if pending { + continue_pendings.push(i); + continue; + } + if let Some((block, col)) = input { + let rows = R::from_column(&col, &self.sort_desc)?; + let cursor = Cursor::new(i, rows); + self.heap.push(Reverse(cursor)); + self.buffer[i] = block; + } + } + self.pending_streams.extend(continue_pendings); + Ok(()) + } + + /// To evaluate the current cursor, and update the top of the heap if necessary. + /// This method can only be called when iterating the heap. + /// + /// Return `true` if the batch is full (need to output). + #[inline(always)] + fn evaluate_cursor(&mut self, mut cursor: Cursor) -> bool { + let max_rows = self.limit.unwrap_or(self.batch_size).min(self.batch_size); + if self.heap.len() == 1 { + let start = cursor.row_index; + let count = (cursor.num_rows() - start).min(max_rows - self.temp_sorted_num_rows); + self.temp_sorted_num_rows += count; + cursor.row_index += count; + self.temp_output_indices + .push((cursor.input_index, start, count)); + } else { + let next_cursor = &find_bigger_child_of_root(&self.heap).0; + if cursor.last().le(&next_cursor.current()) { + // Short Path: + // If the last row of current block is smaller than the next cursor, + // we can drain the whole block. + let start = cursor.row_index; + let count = (cursor.num_rows() - start).min(max_rows - self.temp_sorted_num_rows); + self.temp_sorted_num_rows += count; + cursor.row_index += count; + self.temp_output_indices + .push((cursor.input_index, start, count)); + } else { + // We copy current cursor for advancing, + // and we will use this copied cursor to update the top of the heap at last + // (let heap adjust itself without popping and pushing any element). + let start = cursor.row_index; + while !cursor.is_finished() + && cursor.le(next_cursor) + && self.temp_sorted_num_rows < max_rows + { + // If the cursor is smaller than the next cursor, don't need to push the cursor back to the heap. + self.temp_sorted_num_rows += 1; + cursor.advance(); + } + self.temp_output_indices.push(( + cursor.input_index, + start, + cursor.row_index - start, + )); + } + } + + if !cursor.is_finished() { + // Update the top of the heap. + // `self.heap.peek_mut` will return a `PeekMut` object which allows us to modify the top element of the heap. + // The heap will adjust itself automatically when the `PeekMut` object is dropped (RAII). + self.heap.peek_mut().unwrap().0 = cursor; + } else { + // Pop the current `cursor`. + self.heap.pop(); + // We have read all rows of this block, need to release the old memory and read a new one. + let temp_block = DataBlock::take_by_slices_limit_from_blocks( + &self.buffer, + &self.temp_output_indices, + None, + ); + self.buffer[cursor.input_index] = DataBlock::empty_with_schema(self.schema.clone()); + self.temp_sorted_blocks.push(temp_block); + self.temp_output_indices.clear(); + self.pending_streams.push_back(cursor.input_index); + } + + debug_assert!(self.temp_sorted_num_rows <= max_rows); + self.temp_sorted_num_rows == max_rows + } + + fn build_output(&mut self) -> Result { + if !self.temp_output_indices.is_empty() { + let block = DataBlock::take_by_slices_limit_from_blocks( + &self.buffer, + &self.temp_output_indices, + None, + ); + self.temp_sorted_blocks.push(block); + } + let block = DataBlock::concat(&self.temp_sorted_blocks)?; + + debug_assert_eq!(block.num_rows(), self.temp_sorted_num_rows); + debug_assert!(block.num_rows() <= self.batch_size); + + self.limit = self.limit.map(|limit| limit - self.temp_sorted_num_rows); + self.temp_sorted_blocks.clear(); + self.temp_output_indices.clear(); + self.temp_sorted_num_rows = 0; + + Ok(block) + } + + /// Returns the next sorted block and if it is pending. + /// + /// If the block is [None], it means the merger is finished or pending (has pending streams). + pub fn next_block(&mut self) -> Result> { + if self.is_finished() { + return Ok(None); + } + + if self.has_pending_stream() { + self.poll_pending_stream()?; + if self.has_pending_stream() { + return Ok(None); + } + } + + // No pending streams now. + if self.heap.is_empty() { + return if self.temp_sorted_num_rows > 0 { + Ok(Some(self.build_output()?)) + } else { + Ok(None) + }; + } + + while let Some(Reverse(cursor)) = self.heap.peek() { + if self.evaluate_cursor(cursor.clone()) { + break; + } + if self.has_pending_stream() { + self.poll_pending_stream()?; + if self.has_pending_stream() { + return Ok(None); + } + } + } + + Ok(Some(self.build_output()?)) + } + + /// The async version of `next_block`. + pub async fn async_next_block(&mut self) -> Result> { + if self.is_finished() { + return Ok(None); + } + + if self.has_pending_stream() { + self.async_poll_pending_stream().await?; + if self.has_pending_stream() { + return Ok(None); + } + } + + // No pending streams now. + if self.heap.is_empty() { + return if self.temp_sorted_num_rows > 0 { + Ok(Some(self.build_output()?)) + } else { + Ok(None) + }; + } + + while let Some(Reverse(cursor)) = self.heap.peek() { + if self.evaluate_cursor(cursor.clone()) { + break; + } + if self.has_pending_stream() { + self.async_poll_pending_stream().await?; + if self.has_pending_stream() { + return Ok(None); + } + } + } + + Ok(Some(self.build_output()?)) + } +} diff --git a/src/query/pipeline/transforms/src/processors/transforms/sort/mod.rs b/src/query/pipeline/transforms/src/processors/transforms/sort/mod.rs index a417cba75819..edcf7f7c668a 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sort/mod.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sort/mod.rs @@ -13,10 +13,12 @@ // limitations under the License. mod cursor; +mod merger; mod rows; mod spill; pub mod utils; pub use cursor::*; +pub use merger::*; pub use rows::*; pub use spill::*; diff --git a/src/query/pipeline/transforms/src/processors/transforms/sort/rows/common.rs b/src/query/pipeline/transforms/src/processors/transforms/sort/rows/common.rs index 6405fb0b30a6..fcfbe291073d 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sort/rows/common.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sort/rows/common.rs @@ -48,9 +48,13 @@ impl Rows for StringColumn { Column::String(self.clone()) } - fn from_column(col: Column, _: &[SortColumnDescription]) -> Option { + fn try_from_column(col: &Column, _: &[SortColumnDescription]) -> Option { col.as_string().cloned() } + + fn data_type() -> DataType { + DataType::String + } } impl RowConverter for CommonRowConverter { diff --git a/src/query/pipeline/transforms/src/processors/transforms/sort/rows/mod.rs b/src/query/pipeline/transforms/src/processors/transforms/sort/rows/mod.rs index 9437f466c7e0..68892847513c 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sort/rows/mod.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sort/rows/mod.rs @@ -16,7 +16,9 @@ mod common; mod simple; pub use common::*; +use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::types::DataType; use databend_common_expression::BlockEntry; use databend_common_expression::Column; use databend_common_expression::DataSchemaRef; @@ -44,7 +46,19 @@ where Self: Sized + Clone fn len(&self) -> usize; fn row(&self, index: usize) -> Self::Item<'_>; fn to_column(&self) -> Column; - fn from_column(col: Column, desc: &[SortColumnDescription]) -> Option; + + fn from_column(col: &Column, desc: &[SortColumnDescription]) -> Result { + Self::try_from_column(col, desc).ok_or_else(|| { + ErrorCode::BadDataValueType(format!( + "Order column type mismatched. Expecetd {} but got {}", + Self::data_type(), + col.data_type() + )) + }) + } + fn try_from_column(col: &Column, desc: &[SortColumnDescription]) -> Option; + + fn data_type() -> DataType; fn is_empty(&self) -> bool { self.len() == 0 diff --git a/src/query/pipeline/transforms/src/processors/transforms/sort/rows/simple.rs b/src/query/pipeline/transforms/src/processors/transforms/sort/rows/simple.rs index 88367d1f9e70..19314173bbfc 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sort/rows/simple.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sort/rows/simple.rs @@ -18,6 +18,7 @@ use std::marker::PhantomData; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::ArgType; +use databend_common_expression::types::DataType; use databend_common_expression::types::DateType; use databend_common_expression::types::StringType; use databend_common_expression::types::TimestampType; @@ -93,7 +94,7 @@ where impl Rows for SimpleRows where - T: ValueType, + T: ArgType, T::Scalar: Ord, { type Item<'a> = SimpleRow; @@ -114,13 +115,17 @@ where T::upcast_column(self.inner.clone()) } - fn from_column(col: Column, desc: &[SortColumnDescription]) -> Option { - let inner = T::try_downcast_column(&col)?; + fn try_from_column(col: &Column, desc: &[SortColumnDescription]) -> Option { + let inner = T::try_downcast_column(col)?; Some(Self { inner, desc: !desc[0].asc, }) } + + fn data_type() -> DataType { + T::data_type() + } } pub type DateConverter = SimpleRowConverter; diff --git a/src/query/pipeline/transforms/src/processors/transforms/sort/utils.rs b/src/query/pipeline/transforms/src/processors/transforms/sort/utils.rs index 56c314935f55..9e466e14138b 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/sort/utils.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/sort/utils.rs @@ -14,6 +14,15 @@ use std::collections::BinaryHeap; +use databend_common_expression::types::DataType; +use databend_common_expression::DataField; +use databend_common_expression::DataSchema; +use databend_common_expression::DataSchemaRef; +use databend_common_expression::DataSchemaRefExt; +use databend_common_expression::SortColumnDescription; + +pub const ORDER_COL_NAME: &str = "_order_col"; + /// Find the bigger child of the root of the heap. #[inline(always)] pub fn find_bigger_child_of_root(heap: &BinaryHeap) -> &T { @@ -25,3 +34,32 @@ pub fn find_bigger_child_of_root(heap: &BinaryHeap) -> &T { (&slice[1]).max(&slice[2]) } } + +#[inline(always)] +fn order_field_type(schema: &DataSchema, desc: &[SortColumnDescription]) -> DataType { + debug_assert!(!desc.is_empty()); + if desc.len() == 1 { + let order_by_field = schema.field(desc[0].offset); + if matches!( + order_by_field.data_type(), + DataType::Number(_) | DataType::Date | DataType::Timestamp | DataType::String + ) { + return order_by_field.data_type().clone(); + } + } + DataType::String +} + +#[inline(always)] +pub fn add_order_field(schema: DataSchemaRef, desc: &[SortColumnDescription]) -> DataSchemaRef { + if let Some(f) = schema.fields.last() && f.name() == ORDER_COL_NAME { + schema + } else { + let mut fields = schema.fields().clone(); + fields.push(DataField::new( + ORDER_COL_NAME, + order_field_type(&schema, desc), + )); + DataSchemaRefExt::create(fields) + } +} diff --git a/src/query/pipeline/transforms/src/processors/transforms/transform_multi_sort_merge.rs b/src/query/pipeline/transforms/src/processors/transforms/transform_multi_sort_merge.rs index 5de1dd6daea7..69124479fbf0 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/transform_multi_sort_merge.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/transform_multi_sort_merge.rs @@ -512,17 +512,7 @@ where R: Rows + Send + 'static continue; } let mut block = block.convert_to_full(); - let order_col = block - .columns() - .last() - .unwrap() - .value - .as_column() - .unwrap() - .clone(); - let rows = R::from_column(order_col, &self.sort_desc).ok_or_else(|| { - ErrorCode::BadDataValueType("Order column type mismatched.") - })?; + let rows = R::from_column(block.get_last_column(), &self.sort_desc)?; // Remove the order column if self.remove_order_col { block.pop_columns(1); diff --git a/src/query/pipeline/transforms/src/processors/transforms/transform_sort_merge.rs b/src/query/pipeline/transforms/src/processors/transforms/transform_sort_merge.rs index 729b279ed117..cc2bee874477 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/transform_sort_merge.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/transform_sort_merge.rs @@ -12,9 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::cmp::Reverse; -use std::collections::BinaryHeap; use std::intrinsics::unlikely; +use std::marker::PhantomData; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; use std::sync::Arc; @@ -24,16 +23,18 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::row::RowConverter as CommonConverter; use databend_common_expression::BlockMetaInfo; +use databend_common_expression::Column; use databend_common_expression::DataBlock; use databend_common_expression::DataSchemaRef; use databend_common_expression::SortColumnDescription; -use super::sort::utils::find_bigger_child_of_root; use super::sort::CommonRows; use super::sort::Cursor; use super::sort::DateConverter; use super::sort::DateRows; +use super::sort::HeapMerger; use super::sort::Rows; +use super::sort::SortedStream; use super::sort::StringConverter; use super::sort::StringRows; use super::sort::TimestampConverter; @@ -52,9 +53,11 @@ const SPILL_BATCH_BYTES_SIZE: usize = 8 * 1024 * 1024; /// /// For merge sort with limit, see [`super::transform_sort_merge_limit`] pub struct TransformSortMerge { + schema: DataSchemaRef, + sort_desc: Arc>, + block_size: usize, - heap: BinaryHeap>>, - buffer: Vec, + buffer: Vec>, aborting: Arc, // The following fields are used for spilling. @@ -71,18 +74,23 @@ pub struct TransformSortMerge { spill_batch_size: usize, /// The number of spilled blocks in each merge of the spill processor. spill_num_merge: usize, + + _r: PhantomData, } impl TransformSortMerge { pub fn create( + schema: DataSchemaRef, + sort_desc: Arc>, block_size: usize, max_memory_usage: usize, spilling_bytes_threshold: usize, ) -> Self { let may_spill = max_memory_usage != 0 && spilling_bytes_threshold != 0; TransformSortMerge { + schema, + sort_desc, block_size, - heap: BinaryHeap::new(), buffer: vec![], aborting: Arc::new(AtomicBool::new(false)), may_spill, @@ -92,6 +100,7 @@ impl TransformSortMerge { num_rows: 0, spill_batch_size: 0, spill_num_merge: 0, + _r: PhantomData, } } } @@ -112,8 +121,7 @@ impl MergeSort for TransformSortMerge { self.num_bytes += block.memory_size(); self.num_rows += block.num_rows(); - self.buffer.push(block); - self.heap.push(Reverse(init_cursor)); + self.buffer.push(Some((block, init_cursor.to_column()))); if self.may_spill && (self.num_bytes >= self.spilling_bytes_threshold @@ -130,9 +138,9 @@ impl MergeSort for TransformSortMerge { if self.spill_num_merge > 0 { debug_assert!(self.spill_batch_size > 0); // Make the last block as a big memory block. - self.drain_heap(usize::MAX) + self.merge_sort(usize::MAX) } else { - self.drain_heap(self.block_size) + self.merge_sort(self.block_size) } } @@ -159,7 +167,7 @@ impl TransformSortMerge { debug_assert!(self.spill_num_merge > 0); } - let mut blocks = self.drain_heap(self.spill_batch_size)?; + let mut blocks = self.merge_sort(self.spill_batch_size)?; if let Some(b) = blocks.first_mut() { b.replace_meta(spill_meta); } @@ -167,7 +175,6 @@ impl TransformSortMerge { b.replace_meta(Box::new(SortSpillMeta {})); } - debug_assert!(self.heap.is_empty()); self.num_rows = 0; self.num_bytes = 0; self.buffer.clear(); @@ -175,96 +182,60 @@ impl TransformSortMerge { Ok(blocks) } - fn drain_heap(&mut self, batch_size: usize) -> Result> { - // TODO: the codes is highly duplicated with the codes in `transform_sort_spill.rs`, - // need to refactor and merge them later. - if self.num_rows == 0 { + fn merge_sort(&mut self, batch_size: usize) -> Result> { + if self.buffer.is_empty() { return Ok(vec![]); } - let output_block_num = self.num_rows.div_ceil(batch_size); - let mut output_blocks = Vec::with_capacity(output_block_num); - let mut output_indices = Vec::with_capacity(output_block_num); - - // 1. Drain the heap - let mut temp_num_rows = 0; - let mut temp_indices = Vec::new(); - while let Some(Reverse(cursor)) = self.heap.peek() { - if unlikely(self.aborting.load(Ordering::Relaxed)) { - return Err(ErrorCode::AbortedQuery( - "Aborted query, because the server is shutting down or the query was killed.", - )); - } + let size_hint = self.num_rows.div_ceil(batch_size); - let mut cursor = cursor.clone(); - if self.heap.len() == 1 { - let start = cursor.row_index; - let count = (cursor.num_rows() - start).min(batch_size - temp_num_rows); - temp_num_rows += count; - cursor.row_index += count; - temp_indices.push((cursor.input_index, start, count)); - } else { - let next_cursor = &find_bigger_child_of_root(&self.heap).0; - if cursor.last().le(&next_cursor.current()) { - // Short Path: - // If the last row of current block is smaller than the next cursor, - // we can drain the whole block. - let start = cursor.row_index; - let count = (cursor.num_rows() - start).min(batch_size - temp_num_rows); - temp_num_rows += count; - cursor.row_index += count; - temp_indices.push((cursor.input_index, start, count)); - } else { - // We copy current cursor for advancing, - // and we will use this copied cursor to update the top of the heap at last - // (let heap adjust itself without popping and pushing any element). - let start = cursor.row_index; - while !cursor.is_finished() - && cursor.le(next_cursor) - && temp_num_rows < batch_size - { - // If the cursor is smaller than the next cursor, don't need to push the cursor back to the heap. - temp_num_rows += 1; - cursor.advance(); - } - temp_indices.push((cursor.input_index, start, cursor.row_index - start)); - } + if self.buffer.len() == 1 { + // If there is only one block, we don't need to merge. + let (block, _) = self.buffer.pop().unwrap().unwrap(); + let num_rows = block.num_rows(); + if size_hint == 1 { + return Ok(vec![block]); } - - if !cursor.is_finished() { - // Update the top of the heap. - // `self.heap.peek_mut` will return a `PeekMut` object which allows us to modify the top element of the heap. - // The heap will adjust itself automatically when the `PeekMut` object is dropped (RAII). - self.heap.peek_mut().unwrap().0 = cursor; - } else { - // Pop the current `cursor`. - self.heap.pop(); + let mut result = Vec::with_capacity(size_hint); + for i in 0..size_hint { + let start = i * batch_size; + let end = ((i + 1) * batch_size).min(num_rows); + let block = block.slice(start..end); + result.push(block); } - - if temp_num_rows == batch_size { - output_indices.push(temp_indices.clone()); - temp_indices.clear(); - temp_num_rows = 0; - } - } - - if !temp_indices.is_empty() { - output_indices.push(temp_indices); + return Ok(result); } - // 2. Build final blocks from `output_indices`. - for indices in output_indices { + let streams = self.buffer.drain(..).collect::>(); + let mut result = Vec::with_capacity(size_hint); + let mut merger = HeapMerger::::create( + self.schema.clone(), + streams, + self.sort_desc.clone(), + batch_size, + None, + ); + + while let Some(block) = merger.next_block()? { if unlikely(self.aborting.load(Ordering::Relaxed)) { return Err(ErrorCode::AbortedQuery( "Aborted query, because the server is shutting down or the query was killed.", )); } - - let block = DataBlock::take_by_slices_limit_from_blocks(&self.buffer, &indices, None); - output_blocks.push(block); + result.push(block); } - Ok(output_blocks) + debug_assert!(merger.is_finished()); + + Ok(result) + } +} + +type BlockStream = Option<(DataBlock, Column)>; + +impl SortedStream for BlockStream { + fn next(&mut self) -> Result<(Option<(DataBlock, Column)>, bool)> { + Ok((self.take(), false)) } } @@ -284,17 +255,18 @@ pub(super) type MergeSortCommon = TransformSortMergeBase; pub fn sort_merge( - data_schema: DataSchemaRef, + schema: DataSchemaRef, block_size: usize, sort_desc: Vec, data_blocks: Vec, ) -> Result> { + let sort_desc = Arc::new(sort_desc); let mut processor = MergeSortCommon::try_create( - data_schema, - Arc::new(sort_desc), + schema.clone(), + sort_desc.clone(), false, false, - MergeSortCommonImpl::create(block_size, 0, 0), + MergeSortCommonImpl::create(schema, sort_desc, block_size, 0, 0), )?; for block in data_blocks { processor.transform(block)?; diff --git a/src/query/pipeline/transforms/src/processors/transforms/transform_sort_merge_base.rs b/src/query/pipeline/transforms/src/processors/transforms/transform_sort_merge_base.rs index b16b7e1eeac3..3a636a284080 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/transform_sort_merge_base.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/transform_sort_merge_base.rs @@ -15,7 +15,6 @@ use std::marker::PhantomData; use std::sync::Arc; -use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; @@ -55,6 +54,7 @@ use super::MergeSortTimestamp; use super::MergeSortTimestampImpl; use super::TransformSortMerge; use super::TransformSortMergeLimit; +use crate::processors::sort::utils::ORDER_COL_NAME; pub enum Status { /// Continue to add blocks. @@ -134,16 +134,7 @@ where fn transform(&mut self, mut block: DataBlock) -> Result> { let rows = if self.order_col_generated { - let order_col = block - .columns() - .last() - .unwrap() - .value - .as_column() - .unwrap() - .clone(); - let rows = R::from_column(order_col, &self.sort_desc) - .ok_or_else(|| ErrorCode::BadDataValueType("Order column type mismatched."))?; + let rows = R::from_column(block.get_last_column(), &self.sort_desc)?; if !self.output_order_col { // The next processor could be a sort spill processor which need order column. // And the order column will be removed in that processor. @@ -250,6 +241,12 @@ impl TransformSortMergeBuilder { } pub fn build(self) -> Result> { + debug_assert!(if self.output_order_col { + self.schema.has_field(ORDER_COL_NAME) + } else { + !self.schema.has_field(ORDER_COL_NAME) + }); + if self.limit.is_some() { self.build_sort_merge_limit() } else { @@ -283,14 +280,16 @@ impl TransformSortMergeBuilder { SimpleRows>, SimpleRowConverter>, >::try_create( - schema, - sort_desc, + schema.clone(), + sort_desc.clone(), order_col_generated, output_order_col, TransformSortMerge::create( + schema, + sort_desc, block_size, max_memory_usage, - spilling_bytes_threshold_per_core + spilling_bytes_threshold_per_core, ), )?, ), @@ -299,11 +298,13 @@ impl TransformSortMergeBuilder { input, output, MergeSortDate::try_create( - schema, - sort_desc, + schema.clone(), + sort_desc.clone(), order_col_generated, output_order_col, MergeSortDateImpl::create( + schema, + sort_desc, block_size, max_memory_usage, spilling_bytes_threshold_per_core, @@ -314,11 +315,13 @@ impl TransformSortMergeBuilder { input, output, MergeSortTimestamp::try_create( - schema, - sort_desc, + schema.clone(), + sort_desc.clone(), order_col_generated, output_order_col, MergeSortTimestampImpl::create( + schema, + sort_desc, block_size, max_memory_usage, spilling_bytes_threshold_per_core, @@ -329,11 +332,13 @@ impl TransformSortMergeBuilder { input, output, MergeSortString::try_create( - schema, - sort_desc, + schema.clone(), + sort_desc.clone(), order_col_generated, output_order_col, MergeSortStringImpl::create( + schema, + sort_desc, block_size, max_memory_usage, spilling_bytes_threshold_per_core, @@ -344,11 +349,13 @@ impl TransformSortMergeBuilder { input, output, MergeSortCommon::try_create( - schema, - sort_desc, + schema.clone(), + sort_desc.clone(), order_col_generated, output_order_col, MergeSortCommonImpl::create( + schema, + sort_desc, block_size, max_memory_usage, spilling_bytes_threshold_per_core, @@ -361,11 +368,13 @@ impl TransformSortMergeBuilder { input, output, MergeSortCommon::try_create( - schema, - sort_desc, + schema.clone(), + sort_desc.clone(), order_col_generated, output_order_col, MergeSortCommonImpl::create( + schema, + sort_desc, block_size, max_memory_usage, spilling_bytes_threshold_per_core, diff --git a/src/query/pipeline/transforms/tests/it/main.rs b/src/query/pipeline/transforms/tests/it/main.rs new file mode 100644 index 000000000000..dc8da8592a46 --- /dev/null +++ b/src/query/pipeline/transforms/tests/it/main.rs @@ -0,0 +1,15 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod merger; diff --git a/src/query/pipeline/transforms/tests/it/merger.rs b/src/query/pipeline/transforms/tests/it/merger.rs new file mode 100644 index 000000000000..18c9a81453db --- /dev/null +++ b/src/query/pipeline/transforms/tests/it/merger.rs @@ -0,0 +1,262 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::VecDeque; +use std::sync::Arc; + +use databend_common_base::base::tokio; +use databend_common_exception::Result; +use databend_common_expression::block_debug::pretty_format_blocks; +use databend_common_expression::types::DataType; +use databend_common_expression::types::Int32Type; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::Column; +use databend_common_expression::DataBlock; +use databend_common_expression::DataField; +use databend_common_expression::DataSchemaRefExt; +use databend_common_expression::FromData; +use databend_common_expression::SortColumnDescription; +use databend_common_pipeline_transforms::processors::sort::HeapMerger; +use databend_common_pipeline_transforms::processors::sort::SimpleRows; +use databend_common_pipeline_transforms::processors::sort::SortedStream; +use itertools::Itertools; +use rand::rngs::ThreadRng; +use rand::Rng; + +struct TestStream { + data: VecDeque, + rng: ThreadRng, +} + +unsafe impl Send for TestStream {} + +impl TestStream { + fn new(data: VecDeque) -> Self { + Self { + data, + rng: rand::thread_rng(), + } + } +} + +impl SortedStream for TestStream { + fn next(&mut self) -> Result<(Option<(DataBlock, Column)>, bool)> { + // To simulate the real scenario, we randomly decide whether the stream is pending or not. + let pending = self.rng.gen_bool(0.5); + if pending { + Ok((None, true)) + } else { + Ok(( + self.data.pop_front().map(|b| { + let col = b.get_last_column().clone(); + (b, col) + }), + false, + )) + } + } +} + +type TestMerger = HeapMerger, TestStream>; + +fn prepare_input_and_result( + data: Vec>>, + limit: Option, +) -> (Vec>, DataBlock) { + let input = data + .clone() + .into_iter() + .map(|v| { + v.into_iter() + .map(|v| DataBlock::new_from_columns(vec![Int32Type::from_data(v)])) + .collect::>() + }) + .collect::>(); + let result = data + .into_iter() + .flatten() + .flatten() + .sorted() + .take(limit.unwrap_or(usize::MAX)) + .collect::>(); + let result = DataBlock::new_from_columns(vec![Int32Type::from_data(result)]); + + (input, result) +} + +/// Returns (input, expected) +fn basic_test_data(limit: Option) -> (Vec>, DataBlock) { + let data = vec![ + vec![vec![1, 2, 3, 4], vec![4, 5, 6, 7]], + vec![vec![1, 1, 1, 1], vec![1, 10, 100, 2000]], + vec![vec![0, 2, 4, 5]], + ]; + + prepare_input_and_result(data, limit) +} + +/// Returns (input, expected, batch_size, num_merge) +fn random_test_data(rng: &mut ThreadRng) -> (Vec>, DataBlock, Option) { + let random_batch_size = rng.gen_range(1..=10); + let random_num_streams = rng.gen_range(5..=10); + + let random_data = (0..random_num_streams) + .map(|_| { + let random_num_blocks = rng.gen_range(1..=10); + let mut data = (0..random_batch_size * random_num_blocks) + .map(|_| rng.gen_range(0..=1000)) + .collect::>(); + data.sort(); + data.chunks(random_batch_size) + .map(|v| v.to_vec()) + .collect::>() + }) + .collect::>(); + + let num_rows = random_data + .iter() + .map(|v| v.iter().map(|v| v.len()).sum::()) + .sum::(); + let limit = rng.gen_range(0..=num_rows); + let (input, expected) = prepare_input_and_result(random_data, Some(limit)); + (input, expected, Some(limit)) +} + +fn create_test_merger(input: Vec>, limit: Option) -> TestMerger { + let schema = DataSchemaRefExt::create(vec![DataField::new( + "a", + DataType::Number(NumberDataType::Int32), + )]); + let sort_desc = Arc::new(vec![SortColumnDescription { + offset: 0, + asc: true, + nulls_first: true, + is_nullable: false, + }]); + let streams = input + .into_iter() + .map(|v| TestStream::new(v.into_iter().collect::>())) + .collect::>(); + + TestMerger::create(schema, streams, sort_desc, 4, limit) +} + +fn check_result(result: Vec, expected: DataBlock) { + if expected.is_empty() { + if !result.is_empty() && !DataBlock::concat(&result).unwrap().is_empty() { + panic!( + "\nexpected empty block, but got:\n {}", + pretty_format_blocks(&result).unwrap() + ) + } + return; + } + + let result_rows = result.iter().map(|v| v.num_rows()).sum::(); + let result = pretty_format_blocks(&result).unwrap(); + let expected_rows = expected.num_rows(); + let expected = pretty_format_blocks(&[expected]).unwrap(); + assert_eq!( + expected, result, + "\nexpected (num_rows = {}):\n{}\nactual (num_rows = {}):\n{}", + expected_rows, expected, result_rows, result + ); +} + +fn test(mut merger: TestMerger, expected: DataBlock) -> Result<()> { + let mut result = Vec::new(); + + while !merger.is_finished() { + if let Some(block) = merger.next_block()? { + result.push(block); + } + } + + check_result(result, expected); + + Ok(()) +} + +async fn async_test(mut merger: TestMerger, expected: DataBlock) -> Result<()> { + let mut result = Vec::new(); + + while !merger.is_finished() { + if let Some(block) = merger.async_next_block().await? { + result.push(block); + } + } + check_result(result, expected); + + Ok(()) +} + +fn test_basic(limit: Option) -> Result<()> { + let (input, expected) = basic_test_data(limit); + let merger = create_test_merger(input, limit); + test(merger, expected) +} + +async fn async_test_basic(limit: Option) -> Result<()> { + let (input, expected) = basic_test_data(limit); + let merger = create_test_merger(input, limit); + async_test(merger, expected).await +} + +#[test] +fn test_basic_with_limit() -> Result<()> { + test_basic(None)?; + test_basic(Some(0))?; + test_basic(Some(1))?; + test_basic(Some(5))?; + test_basic(Some(20))?; + test_basic(Some(21))?; + test_basic(Some(1000000)) +} + +#[tokio::test(flavor = "multi_thread")] +async fn async_test_basic_with_limit() -> Result<()> { + async_test_basic(None).await?; + async_test_basic(Some(0)).await?; + async_test_basic(Some(1)).await?; + async_test_basic(Some(5)).await?; + async_test_basic(Some(20)).await?; + async_test_basic(Some(21)).await?; + async_test_basic(Some(1000000)).await +} + +#[test] +fn test_fuzz() -> Result<()> { + let mut rng = rand::thread_rng(); + + for _ in 0..10 { + let (input, expected, limit) = random_test_data(&mut rng); + let merger = create_test_merger(input, limit); + test(merger, expected)?; + } + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_fuzz_async() -> Result<()> { + let mut rng = rand::thread_rng(); + + for _ in 0..10 { + let (input, expected, limit) = random_test_data(&mut rng); + let merger = create_test_merger(input, limit); + async_test(merger, expected).await?; + } + + Ok(()) +} diff --git a/src/query/service/src/pipelines/builders/builder_sort.rs b/src/query/service/src/pipelines/builders/builder_sort.rs index b921b8197695..47f7a087ae75 100644 --- a/src/query/service/src/pipelines/builders/builder_sort.rs +++ b/src/query/service/src/pipelines/builders/builder_sort.rs @@ -15,15 +15,12 @@ use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::types::DataType; -use databend_common_expression::DataField; -use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRef; -use databend_common_expression::DataSchemaRefExt; use databend_common_expression::SortColumnDescription; use databend_common_pipeline_core::processors::ProcessorPtr; use databend_common_pipeline_core::query_spill_prefix; use databend_common_pipeline_core::Pipeline; +use databend_common_pipeline_transforms::processors::sort::utils::add_order_field; use databend_common_pipeline_transforms::processors::try_add_multi_sort_merge; use databend_common_pipeline_transforms::processors::ProcessorProfileWrapper; use databend_common_pipeline_transforms::processors::TransformSortMergeBuilder; @@ -76,24 +73,24 @@ impl PipelineBuilder { } } - let input_schema = sort.output_schema()?; + let plan_schema = sort.output_schema()?; let sort_desc = sort .order_by .iter() .map(|desc| { - let offset = input_schema.index_of(&desc.order_by.to_string())?; + let offset = plan_schema.index_of(&desc.order_by.to_string())?; Ok(SortColumnDescription { offset, asc: desc.asc, nulls_first: desc.nulls_first, - is_nullable: input_schema.field(offset).is_nullable(), // This information is not needed here. + is_nullable: plan_schema.field(offset).is_nullable(), // This information is not needed here. }) }) .collect::>>()?; self.build_sort_pipeline( - input_schema, + plan_schema, sort_desc, sort.plan_id, sort.limit, @@ -103,7 +100,7 @@ impl PipelineBuilder { pub(crate) fn build_sort_pipeline( &mut self, - input_schema: DataSchemaRef, + plan_schema: DataSchemaRef, sort_desc: Vec, plan_id: u32, limit: Option, @@ -124,7 +121,7 @@ impl PipelineBuilder { }; let mut builder = - SortPipelineBuilder::create(self.ctx.clone(), input_schema.clone(), sort_desc.clone()) + SortPipelineBuilder::create(self.ctx.clone(), plan_schema.clone(), sort_desc.clone()) .with_partial_block_size(block_size) .with_final_block_size(block_size) .with_limit(limit) @@ -139,7 +136,7 @@ impl PipelineBuilder { if self.main_pipeline.output_len() > 1 { try_add_multi_sort_merge( &mut self.main_pipeline, - input_schema, + plan_schema, block_size, limit, sort_desc, @@ -293,11 +290,17 @@ impl SortPipelineBuilder { let may_spill = max_memory_usage != 0 && bytes_limit_per_proc != 0; + let sort_merge_output_schema = if output_order_col || may_spill { + add_order_field(self.schema.clone(), &self.sort_desc) + } else { + self.schema.clone() + }; + pipeline.add_transform(|input, output| { let builder = TransformSortMergeBuilder::create( input, output, - self.schema.clone(), + sort_merge_output_schema.clone(), self.sort_desc.clone(), self.partial_block_size, ) @@ -320,18 +323,8 @@ impl SortPipelineBuilder { })?; if may_spill { + let schema = add_order_field(sort_merge_output_schema.clone(), &self.sort_desc); let config = SpillerConfig::create(query_spill_prefix(&self.ctx.get_tenant())); - // The input of the processor must contain an order column. - let schema = if let Some(f) = self.schema.fields.last() && f.name() == "_order_col" { - self.schema.clone() - } else { - let mut fields = self.schema.fields().clone(); - fields.push(DataField::new( - "_order_col", - order_column_type(&self.sort_desc, &self.schema), - )); - DataSchemaRefExt::create(fields) - }; pipeline.add_transform(|input, output| { let op = DataOperator::instance().operator(); let spiller = @@ -360,7 +353,7 @@ impl SortPipelineBuilder { // Multi-pipelines merge sort try_add_multi_sort_merge( pipeline, - self.schema, + self.schema.clone(), self.final_block_size, self.limit, self.sort_desc, @@ -372,17 +365,3 @@ impl SortPipelineBuilder { Ok(()) } } - -fn order_column_type(desc: &[SortColumnDescription], schema: &DataSchema) -> DataType { - debug_assert!(!desc.is_empty()); - if desc.len() == 1 { - let order_by_field = schema.field(desc[0].offset); - if matches!( - order_by_field.data_type(), - DataType::Number(_) | DataType::Date | DataType::Timestamp | DataType::String - ) { - return order_by_field.data_type().clone(); - } - } - DataType::String -} diff --git a/src/query/service/src/pipelines/processors/transforms/transform_sort_spill.rs b/src/query/service/src/pipelines/processors/transforms/transform_sort_spill.rs index 4b1f6b42d92c..1f71989ee29b 100644 --- a/src/query/service/src/pipelines/processors/transforms/transform_sort_spill.rs +++ b/src/query/service/src/pipelines/processors/transforms/transform_sort_spill.rs @@ -13,20 +13,18 @@ // limitations under the License. use std::any::Any; -use std::cmp::Reverse; -use std::collections::BinaryHeap; use std::collections::VecDeque; use std::marker::PhantomData; use std::sync::Arc; use std::time::Instant; -use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberType; use databend_common_expression::with_number_mapped_type; use databend_common_expression::BlockMetaInfoDowncast; +use databend_common_expression::Column; use databend_common_expression::DataBlock; use databend_common_expression::DataSchemaRef; use databend_common_expression::SortColumnDescription; @@ -40,14 +38,14 @@ use databend_common_pipeline_core::processors::Event; use databend_common_pipeline_core::processors::InputPort; use databend_common_pipeline_core::processors::OutputPort; use databend_common_pipeline_core::processors::Processor; -use databend_common_pipeline_transforms::processors::sort::utils::find_bigger_child_of_root; use databend_common_pipeline_transforms::processors::sort::CommonRows; -use databend_common_pipeline_transforms::processors::sort::Cursor; use databend_common_pipeline_transforms::processors::sort::DateRows; +use databend_common_pipeline_transforms::processors::sort::HeapMerger; use databend_common_pipeline_transforms::processors::sort::Rows; use databend_common_pipeline_transforms::processors::sort::SimpleRows; use databend_common_pipeline_transforms::processors::sort::SortSpillMeta; use databend_common_pipeline_transforms::processors::sort::SortSpillMetaWithParams; +use databend_common_pipeline_transforms::processors::sort::SortedStream; use databend_common_pipeline_transforms::processors::sort::StringRows; use databend_common_pipeline_transforms::processors::sort::TimestampRows; @@ -326,15 +324,16 @@ where R: Rows + Sync + Send + 'static streams.push(stream); } - let mut merger = Merger::::create( + let mut merger = HeapMerger::::create( self.schema.clone(), streams, self.sort_desc.clone(), self.batch_size, + None, ); let mut spilled = VecDeque::new(); - while let Some(block) = merger.next().await? { + while let Some(block) = merger.async_next_block().await? { let ins = Instant::now(); let (location, bytes) = self.spiller.spill_block(block).await?; @@ -347,6 +346,8 @@ where R: Rows + Sync + Send + 'static spilled.push_back(location); } + + debug_assert!(merger.is_finished()); self.unmerged_blocks.push_back(spilled); Ok(()) @@ -358,8 +359,9 @@ enum BlockStream { Block(Option), } -impl BlockStream { - async fn next(&mut self) -> Result> { +#[async_trait::async_trait] +impl SortedStream for BlockStream { + async fn async_next(&mut self) -> Result<(Option<(DataBlock, Column)>, bool)> { let block = match self { BlockStream::Block(block) => block.take(), BlockStream::Spilled((files, spiller)) => { @@ -380,149 +382,13 @@ impl BlockStream { } } }; - Ok(block) - } -} - -/// A merge sort operator to merge multiple sorted streams. -/// -/// TODO: reuse this operator in other places such as `TransformMultiSortMerge` and `TransformSortMerge`. -struct Merger { - schema: DataSchemaRef, - sort_desc: Arc>, - unsorted_streams: Vec, - heap: BinaryHeap>>, - buffer: Vec, - pending_stream: VecDeque, - batch_size: usize, -} - -impl Merger { - fn create( - schema: DataSchemaRef, - streams: Vec, - sort_desc: Arc>, - batch_size: usize, - ) -> Self { - // We only create a merger when there are at least two streams. - debug_assert!(streams.len() > 1); - let heap = BinaryHeap::with_capacity(streams.len()); - let buffer = vec![DataBlock::empty_with_schema(schema.clone()); streams.len()]; - let pending_stream = (0..streams.len()).collect(); - - Self { - schema, - unsorted_streams: streams, - heap, - buffer, - batch_size, - sort_desc, - pending_stream, - } - } - - // This method can only be called when there is no data of the stream in the heap. - async fn poll_pending_stream(&mut self) -> Result<()> { - while let Some(i) = self.pending_stream.pop_front() { - debug_assert!(self.buffer[i].is_empty()); - if let Some(block) = self.unsorted_streams[i].next().await? { - let order_col = block.columns().last().unwrap().value.as_column().unwrap(); - let rows = R::from_column(order_col.clone(), &self.sort_desc) - .ok_or_else(|| ErrorCode::BadDataValueType("Order column type mismatched."))?; - let cursor = Cursor::new(i, rows); - self.heap.push(Reverse(cursor)); - self.buffer[i] = block; - } - } - Ok(()) - } - - async fn next(&mut self) -> Result> { - if !self.pending_stream.is_empty() { - self.poll_pending_stream().await?; - } - - if self.heap.is_empty() { - return Ok(None); - } - - let mut num_rows = 0; - - // (input_index, row_start, count) - let mut output_indices = Vec::new(); - let mut temp_sorted_blocks = Vec::new(); - - while let Some(Reverse(cursor)) = self.heap.peek() { - let mut cursor = cursor.clone(); - if self.heap.len() == 1 { - let start = cursor.row_index; - let count = (cursor.num_rows() - start).min(self.batch_size - num_rows); - num_rows += count; - cursor.row_index += count; - output_indices.push((cursor.input_index, start, count)); - } else { - let next_cursor = &find_bigger_child_of_root(&self.heap).0; - if cursor.last().le(&next_cursor.current()) { - // Short Path: - // If the last row of current block is smaller than the next cursor, - // we can drain the whole block. - let start = cursor.row_index; - let count = (cursor.num_rows() - start).min(self.batch_size - num_rows); - num_rows += count; - cursor.row_index += count; - output_indices.push((cursor.input_index, start, count)); - } else { - // We copy current cursor for advancing, - // and we will use this copied cursor to update the top of the heap at last - // (let heap adjust itself without popping and pushing any element). - let start = cursor.row_index; - while !cursor.is_finished() - && cursor.le(next_cursor) - && num_rows < self.batch_size - { - // If the cursor is smaller than the next cursor, don't need to push the cursor back to the heap. - num_rows += 1; - cursor.advance(); - } - output_indices.push((cursor.input_index, start, cursor.row_index - start)); - } - } - - if !cursor.is_finished() { - // Update the top of the heap. - // `self.heap.peek_mut` will return a `PeekMut` object which allows us to modify the top element of the heap. - // The heap will adjust itself automatically when the `PeekMut` object is dropped (RAII). - self.heap.peek_mut().unwrap().0 = cursor; - } else { - // Pop the current `cursor`. - self.heap.pop(); - // We have read all rows of this block, need to release the old memory and read a new one. - let temp_block = DataBlock::take_by_slices_limit_from_blocks( - &self.buffer, - &output_indices, - None, - ); - self.buffer[cursor.input_index] = DataBlock::empty_with_schema(self.schema.clone()); - temp_sorted_blocks.push(temp_block); - output_indices.clear(); - self.pending_stream.push_back(cursor.input_index); - self.poll_pending_stream().await?; - } - - if num_rows == self.batch_size { - break; - } - } - - if !output_indices.is_empty() { - let block = - DataBlock::take_by_slices_limit_from_blocks(&self.buffer, &output_indices, None); - temp_sorted_blocks.push(block); - } - - let block = DataBlock::concat(&temp_sorted_blocks)?; - debug_assert!(block.num_rows() <= self.batch_size); - Ok(Some(block)) + Ok(( + block.map(|b| { + let col = b.get_last_column().clone(); + (b, col) + }), + false, + )) } } @@ -612,6 +478,7 @@ mod tests { use databend_common_pipeline_core::processors::InputPort; use databend_common_pipeline_core::processors::OutputPort; use databend_common_pipeline_transforms::processors::sort::SimpleRows; + use databend_common_pipeline_transforms::processors::sort::SortedStream; use databend_common_storage::DataOperator; use itertools::Itertools; use rand::rngs::ThreadRng; @@ -737,7 +604,8 @@ mod tests { )); let mut result = Vec::new(); - while let Some(block) = block_stream.next().await? { + + while let (Some((block, _)), _) = block_stream.async_next().await? { result.push(block); } diff --git a/src/query/sql/src/executor/physical_plans/physical_sort.rs b/src/query/sql/src/executor/physical_plans/physical_sort.rs index 67c940b8af88..0a9714ef869f 100644 --- a/src/query/sql/src/executor/physical_plans/physical_sort.rs +++ b/src/query/sql/src/executor/physical_plans/physical_sort.rs @@ -18,6 +18,7 @@ use databend_common_expression::DataField; use databend_common_expression::DataSchema; use databend_common_expression::DataSchemaRef; use databend_common_expression::DataSchemaRefExt; +use databend_common_pipeline_transforms::processors::sort::utils::ORDER_COL_NAME; use itertools::Itertools; use crate::executor::explain::PlanStatsInfo; @@ -65,7 +66,7 @@ impl Sort { if matches!(self.after_exchange, Some(true)) { // If the plan is after exchange plan in cluster mode, // the order column is at the last of the input schema. - debug_assert_eq!(fields.last().unwrap().name(), "_order_col"); + debug_assert_eq!(fields.last().unwrap().name(), ORDER_COL_NAME); debug_assert_eq!( fields.last().unwrap().data_type(), &self.order_col_type(&input_schema)? @@ -88,7 +89,7 @@ impl Sort { // If the plan is before exchange plan in cluster mode, // the order column should be added to the output schema. fields.push(DataField::new( - "_order_col", + ORDER_COL_NAME, self.order_col_type(&input_schema)?, )); } From f68b51b2b69a8c2e67eb67f8bdfe8b907215df33 Mon Sep 17 00:00:00 2001 From: junxiangMu <63799833+guojidan@users.noreply.github.com> Date: Mon, 18 Dec 2023 12:39:02 +0800 Subject: [PATCH 03/20] feat: add range for show settings (#14049) --- src/query/sql/src/planner/binder/show.rs | 2 +- .../suites/base/06_show/06_0003_show_settings.test | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/query/sql/src/planner/binder/show.rs b/src/query/sql/src/planner/binder/show.rs index f9bf2914b4fc..6c1b2e12a748 100644 --- a/src/query/sql/src/planner/binder/show.rs +++ b/src/query/sql/src/planner/binder/show.rs @@ -65,7 +65,7 @@ impl Binder { ) -> Result { let (show_limit, limit_str) = get_show_options(show_options, None); let query = format!( - "SELECT name, value, default, level, description, type FROM system.settings {} ORDER BY name {}", + "SELECT name, value, default, `range`, level, description, type FROM system.settings {} ORDER BY name {}", show_limit, limit_str, ); diff --git a/tests/sqllogictests/suites/base/06_show/06_0003_show_settings.test b/tests/sqllogictests/suites/base/06_show/06_0003_show_settings.test index bdc3878c2879..b18ea1520f51 100644 --- a/tests/sqllogictests/suites/base/06_show/06_0003_show_settings.test +++ b/tests/sqllogictests/suites/base/06_show/06_0003_show_settings.test @@ -31,12 +31,12 @@ SHOW SETTINGS LIKE 'enable%' query TT SHOW SETTINGS LIKE 'max%' LIMIT 1 ---- -max_block_size 65536 65536 SESSION Sets the maximum byte size of a single data block that can be read. UInt64 +max_block_size 65536 65536 None SESSION Sets the maximum byte size of a single data block that can be read. UInt64 query TT SHOW SETTINGS WHERE name='max_block_size' LIMIT 1 ---- -max_block_size 65536 65536 SESSION Sets the maximum byte size of a single data block that can be read. UInt64 +max_block_size 65536 65536 None SESSION Sets the maximum byte size of a single data block that can be read. UInt64 statement error SHOW SETTINGS ilike 'ff%' LIMIT 1 From a95649805090801c65e8e7ec2aa4b1201602ea1d Mon Sep 17 00:00:00 2001 From: everpcpc Date: Mon, 18 Dec 2023 12:40:18 +0800 Subject: [PATCH 04/20] fix: multiple otlp loggers conflict (#14050) --- src/common/tracing/src/init.rs | 8 +-- src/common/tracing/src/loggers.rs | 86 ++++++++++++++++--------------- 2 files changed, 48 insertions(+), 46 deletions(-) diff --git a/src/common/tracing/src/init.rs b/src/common/tracing/src/init.rs index 33100d0e28ec..2ec3bcfe6fb5 100644 --- a/src/common/tracing/src/init.rs +++ b/src/common/tracing/src/init.rs @@ -28,8 +28,8 @@ use minitrace::prelude::*; use serde_json::Map; use crate::loggers::new_file_log_writer; -use crate::loggers::new_otlp_log_writer; use crate::loggers::MinitraceLogger; +use crate::loggers::OpenTelemetryLogger; use crate::Config; const HEADER_TRACE_PARENT: &str = "traceparent"; @@ -176,7 +176,7 @@ pub fn init_logging( let mut labels = labels.clone(); labels.insert("category".to_string(), "system".to_string()); labels.extend(cfg.otlp.labels.clone()); - let logger = new_otlp_log_writer(&cfg.otlp.endpoint, labels); + let logger = OpenTelemetryLogger::new(log_name, &cfg.otlp.endpoint, labels); let dispatch = fern::Dispatch::new() .level(cfg.otlp.level.parse().unwrap_or(LevelFilter::Info)) .format(formatter("json")) @@ -211,7 +211,7 @@ pub fn init_logging( let mut labels = labels.clone(); labels.insert("category".to_string(), "query".to_string()); labels.extend(cfg.query.labels.clone()); - let logger = new_otlp_log_writer(&cfg.query.otlp_endpoint, labels); + let logger = OpenTelemetryLogger::new(log_name, &cfg.query.otlp_endpoint, labels); query_logger = query_logger.chain(Box::new(logger) as Box); } } @@ -229,7 +229,7 @@ pub fn init_logging( let mut labels = labels.clone(); labels.insert("category".to_string(), "profile".to_string()); labels.extend(cfg.profile.labels.clone()); - let logger = new_otlp_log_writer(&cfg.profile.otlp_endpoint, labels); + let logger = OpenTelemetryLogger::new(log_name, &cfg.profile.otlp_endpoint, labels); profile_logger = profile_logger.chain(Box::new(logger) as Box); } } diff --git a/src/common/tracing/src/loggers.rs b/src/common/tracing/src/loggers.rs index 1ff32221c8db..7663a9a5d8cd 100644 --- a/src/common/tracing/src/loggers.rs +++ b/src/common/tracing/src/loggers.rs @@ -19,10 +19,10 @@ use std::time::Duration; use std::time::SystemTime; use opentelemetry::logs::AnyValue; -use opentelemetry::logs::Logger as _; +use opentelemetry::logs::Logger; +use opentelemetry::logs::LoggerProvider; use opentelemetry::logs::Severity; use opentelemetry_otlp::WithExportConfig; -use opentelemetry_sdk::logs::Logger; use tracing_appender::non_blocking::NonBlocking; use tracing_appender::non_blocking::WorkerGuard; use tracing_appender::rolling::RollingFileAppender; @@ -96,39 +96,48 @@ impl log::Log for MinitraceLogger { fn flush(&self) {} } -pub(crate) struct OpenTelemetryOTLPLogWriter { - logger: Logger, +pub(crate) struct OpenTelemetryLogger { + logger: opentelemetry_sdk::logs::Logger, + // keep provider alive + provider: opentelemetry_sdk::logs::LoggerProvider, } -pub(crate) fn new_otlp_log_writer( - endpoint: &str, - labels: BTreeMap, -) -> OpenTelemetryOTLPLogWriter { - let kvs = labels - .into_iter() - .map(|(k, v)| opentelemetry::KeyValue::new(k, v)) - .collect::>(); - let log_config = opentelemetry_sdk::logs::Config { - resource: Cow::Owned(opentelemetry::sdk::Resource::new(kvs)), - }; - let export_config = opentelemetry_otlp::ExportConfig { - endpoint: endpoint.to_string(), - protocol: opentelemetry_otlp::Protocol::Grpc, - timeout: Duration::from_secs(opentelemetry_otlp::OTEL_EXPORTER_OTLP_TIMEOUT_DEFAULT), - }; - let exporter = opentelemetry_otlp::new_exporter() - .tonic() - .with_export_config(export_config); - let logger = opentelemetry_otlp::new_pipeline() - .logging() - .with_exporter(exporter) - .with_log_config(log_config) - .install_batch(opentelemetry_sdk::runtime::Tokio) - .expect("install query log otlp pipeline"); - OpenTelemetryOTLPLogWriter { logger } +impl OpenTelemetryLogger { + pub(crate) fn new( + name: impl ToString, + endpoint: &str, + labels: BTreeMap, + ) -> Self { + let kvs = labels + .into_iter() + .map(|(k, v)| opentelemetry::KeyValue::new(k, v)) + .collect::>(); + let export_config = opentelemetry_otlp::ExportConfig { + endpoint: endpoint.to_string(), + protocol: opentelemetry_otlp::Protocol::Grpc, + timeout: Duration::from_secs(opentelemetry_otlp::OTEL_EXPORTER_OTLP_TIMEOUT_DEFAULT), + }; + let exporter_builder: opentelemetry_otlp::LogExporterBuilder = + opentelemetry_otlp::new_exporter() + .tonic() + .with_export_config(export_config) + .into(); + let exporter = exporter_builder + .build_log_exporter() + .expect("build log exporter"); + let provider = opentelemetry_sdk::logs::LoggerProvider::builder() + .with_batch_exporter(exporter, opentelemetry_sdk::runtime::Tokio) + .with_config( + opentelemetry_sdk::logs::Config::default() + .with_resource(opentelemetry_sdk::Resource::new(kvs)), + ) + .build(); + let logger = provider.versioned_logger(name.to_string(), None, None, None); + Self { logger, provider } + } } -impl log::Log for OpenTelemetryOTLPLogWriter { +impl log::Log for OpenTelemetryLogger { fn enabled(&self, _metadata: &log::Metadata<'_>) -> bool { // we handle level and target filter with fern true @@ -144,17 +153,10 @@ impl log::Log for OpenTelemetryOTLPLogWriter { } fn flush(&self) { - match self.logger.provider() { - Some(provider) => { - let result = provider.force_flush(); - for r in result { - if let Err(e) = r { - eprintln!("flush log failed: {}", e); - } - } - } - None => { - eprintln!("flush log failed: logger provider is None"); + let result = self.provider.force_flush(); + for r in result { + if let Err(e) = r { + eprintln!("flush log failed: {}", e); } } } From ce9acb45509a821877f028f1d10ca275a4807ce5 Mon Sep 17 00:00:00 2001 From: Winter Zhang Date: Mon, 18 Dec 2023 12:41:07 +0800 Subject: [PATCH 05/20] chore(executor): use null as root profile parent id (#14051) --- src/query/pipeline/core/src/pipeline.rs | 8 ++++---- src/query/pipeline/core/src/processors/profile.rs | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/query/pipeline/core/src/pipeline.rs b/src/query/pipeline/core/src/pipeline.rs index cd365b3a7eaa..c0ca0f1821d6 100644 --- a/src/query/pipeline/core/src/pipeline.rs +++ b/src/query/pipeline/core/src/pipeline.rs @@ -139,11 +139,11 @@ impl Pipeline { pub fn finalize(mut self) -> Pipeline { for pipe in &mut self.pipes { if let Some(uninitialized_scope) = &mut pipe.scope { - if uninitialized_scope.parent_id == 0 { + if uninitialized_scope.parent_id.is_none() { for (index, scope) in self.plans_scope.iter().enumerate() { if scope.id == uninitialized_scope.id && index != 0 { if let Some(parent_scope) = self.plans_scope.get(index - 1) { - uninitialized_scope.parent_id = parent_scope.id; + uninitialized_scope.parent_id = Some(parent_scope.id); } } } @@ -162,8 +162,8 @@ impl Pipeline { // set the parent node in 'add_pipe' helps skip empty plans(no pipeline). for pipe in &mut self.pipes { if let Some(children) = &mut pipe.scope { - if children.parent_id == 0 && children.id != scope.id { - children.parent_id = scope.id; + if children.parent_id.is_none() && children.id != scope.id { + children.parent_id = Some(scope.id); } } } diff --git a/src/query/pipeline/core/src/processors/profile.rs b/src/query/pipeline/core/src/processors/profile.rs index 3c8bac4975df..d5275bb238d4 100644 --- a/src/query/pipeline/core/src/processors/profile.rs +++ b/src/query/pipeline/core/src/processors/profile.rs @@ -44,7 +44,7 @@ impl Profile { wait_time: AtomicU64::new(0), plan_id: scope.as_ref().map(|x| x.id), plan_name: scope.as_ref().map(|x| x.name.clone()), - plan_parent_id: scope.as_ref().map(|x| x.parent_id), + plan_parent_id: scope.as_ref().and_then(|x| x.parent_id), } } } @@ -109,14 +109,14 @@ impl Drop for PlanScopeGuard { pub struct PlanScope { pub id: u32, pub name: String, - pub parent_id: u32, + pub parent_id: Option, } impl PlanScope { pub fn create(id: u32, name: String) -> PlanScope { PlanScope { id, - parent_id: 0, + parent_id: None, name, } } From 0485c9e01bb3e1fdca4cb9b3e63dbf1fcb756cbd Mon Sep 17 00:00:00 2001 From: "xudong.w" Date: Mon, 18 Dec 2023 14:23:29 +0800 Subject: [PATCH 06/20] chore: reduce tpcds Q2 result set (#14055) --- tests/sqllogictests/suites/tpcds/queries.test | 1776 +---------------- 1 file changed, 1 insertion(+), 1775 deletions(-) diff --git a/tests/sqllogictests/suites/tpcds/queries.test b/tests/sqllogictests/suites/tpcds/queries.test index 042890ab6a4d..23e33f18774d 100644 --- a/tests/sqllogictests/suites/tpcds/queries.test +++ b/tests/sqllogictests/suites/tpcds/queries.test @@ -213,7 +213,7 @@ FROM WHERE date_dim.d_week_seq = wswscs.d_week_seq AND d_year = 2001+1) z WHERE d_week_seq1 = d_week_seq2-53 -ORDER BY d_week_seq1 NULLS FIRST; +ORDER BY d_week_seq1 NULLS FIRST limit 200; ---- 5270 NULL 2.60 NULL 0.71 2.48 NULL NULL 5270 NULL 2.60 NULL 0.71 2.48 NULL NULL @@ -415,1780 +415,6 @@ ORDER BY d_week_seq1 NULLS FIRST; 5274 NULL NULL NULL NULL 0.54 NULL NULL 5274 NULL NULL NULL NULL 0.54 NULL NULL 5274 NULL NULL NULL NULL 0.54 NULL NULL -5274 NULL NULL NULL NULL 0.54 NULL NULL -5274 NULL NULL NULL NULL 0.54 NULL NULL -5274 NULL NULL NULL NULL 0.54 NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5275 NULL NULL 0.71 NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5277 0.74 NULL NULL NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5278 NULL NULL 0.55 NULL NULL NULL NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5279 NULL 1.24 0.95 NULL NULL 0.51 NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5280 0.53 1.66 NULL NULL 0.41 NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5282 NULL NULL NULL 0.17 NULL NULL NULL -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5284 NULL NULL 2.98 2.93 NULL 0.69 1.95 -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5285 0.80 0.71 NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5286 1.62 NULL NULL NULL NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5287 NULL NULL NULL 1.90 NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5288 1.17 NULL NULL NULL NULL NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5291 NULL 0.62 NULL NULL 1.15 NULL NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5292 NULL NULL NULL NULL NULL 0.61 NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5293 1.43 NULL NULL NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5294 NULL NULL 0.69 NULL NULL NULL NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5296 NULL NULL NULL NULL NULL 0.59 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5300 NULL 0.93 NULL NULL NULL 2.01 NULL -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5301 NULL 0.81 2.66 0.70 NULL 1.91 0.07 -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5302 NULL NULL NULL 0.18 NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5304 NULL NULL 3.64 NULL NULL NULL NULL -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5305 0.97 2.21 1.20 NULL 1.05 NULL 0.41 -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5306 NULL 1.92 NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5307 1.86 NULL NULL NULL NULL NULL NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5308 2.23 NULL NULL 1.78 NULL 1.24 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5309 NULL 0.35 0.17 0.14 NULL 1.28 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5310 1.56 4.23 NULL NULL NULL 1.27 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5312 NULL NULL NULL NULL NULL 2.39 NULL -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5313 0.98 NULL 1.79 NULL NULL 3.58 2.41 -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5314 NULL NULL NULL NULL 0.81 NULL NULL -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5315 0.28 NULL 1.05 NULL NULL 1.44 2.86 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5316 5.91 0.46 NULL NULL NULL NULL 0.38 -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5317 NULL 1.09 NULL NULL NULL NULL NULL -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5318 NULL NULL NULL NULL NULL NULL 0.29 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5319 NULL NULL NULL NULL NULL NULL 0.80 -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5320 NULL NULL NULL NULL NULL 1.87 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5321 0.56 0.08 NULL NULL 0.17 0.21 NULL -5322 NULL NULL NULL NULL NULL NULL NULL -5322 NULL NULL NULL NULL NULL NULL NULL -5322 NULL NULL NULL NULL NULL NULL NULL -5322 NULL NULL NULL NULL NULL NULL NULL -5322 NULL NULL NULL NULL NULL NULL NULL -5322 NULL NULL NULL NULL NULL NULL NULL -5322 NULL NULL NULL NULL NULL NULL NULL # Q3 onlyif mysql From 165605c625a52fc6862ad0faa257c780c6b8f144 Mon Sep 17 00:00:00 2001 From: dantengsky Date: Mon, 18 Dec 2023 16:48:33 +0800 Subject: [PATCH 07/20] fix: abort vacuum if there are io execption (#14052) * fix: abort vacuum if there are io execption * add unit test --- .../fuse/operations/vacuum_drop_tables.rs | 46 +++--- .../it/storages/fuse/operations/vacuum.rs | 143 +++++++++++++++++- 2 files changed, 166 insertions(+), 23 deletions(-) diff --git a/src/query/ee/src/storages/fuse/operations/vacuum_drop_tables.rs b/src/query/ee/src/storages/fuse/operations/vacuum_drop_tables.rs index efb2df4daa83..fc8c924e8eaf 100644 --- a/src/query/ee/src/storages/fuse/operations/vacuum_drop_tables.rs +++ b/src/query/ee/src/storages/fuse/operations/vacuum_drop_tables.rs @@ -17,44 +17,33 @@ use std::time::Instant; use databend_common_catalog::table::Table; use databend_common_exception::Result; +use databend_common_meta_app::schema::TableInfo; use databend_common_storages_fuse::FuseTable; use futures_util::TryStreamExt; use log::info; use opendal::EntryMode; use opendal::Metakey; +use opendal::Operator; #[async_backtrace::framed] -async fn do_vacuum_drop_table( - table: Arc, +pub async fn do_vacuum_drop_table( + table_info: &TableInfo, + operator: &Operator, dry_run_limit: Option, ) -> Result>> { - // only operate fuse table - if table.engine() != "FUSE" { - info!( - "ignore table {} not of FUSE engine, table engine {}", - table.get_table_info().name, - table.engine() - ); - return Ok(None); - } - let table_info = table.get_table_info(); // storage_params is_some means it is an external table, ignore if table_info.meta.storage_params.is_some() { - info!("ignore external table {}", table.get_table_info().name); + info!("ignore external table {}", table_info.name); return Ok(None); } - let fuse_table = FuseTable::try_from_table(table.as_ref())?; - - let operator = fuse_table.get_operator_ref(); let dir = format!("{}/", FuseTable::parse_storage_prefix(table_info)?); - info!("vacuum drop table {:?} dir {:?}", table.name(), dir); + info!("vacuum drop table {:?} dir {:?}", table_info.name, dir); let start = Instant::now(); let ret = match dry_run_limit { None => { - let _ = operator.remove_all(&dir).await; - + operator.remove_all(&dir).await?; Ok(None) } Some(dry_run_limit) => { @@ -67,7 +56,7 @@ async fn do_vacuum_drop_table( while let Some(de) = ds.try_next().await? { let meta = de.metadata(); if EntryMode::FILE == meta.mode() { - list_files.push((fuse_table.name().to_string(), de.name().to_string())); + list_files.push((table_info.name.clone(), de.name().to_string())); if list_files.len() >= dry_run_limit { break; } @@ -80,7 +69,7 @@ async fn do_vacuum_drop_table( info!( "vacuum drop table {:?} dir {:?}, cost:{} sec", - table.name(), + table_info.name, dir, start.elapsed().as_secs() ); @@ -98,7 +87,20 @@ pub async fn do_vacuum_drop_tables( let mut list_files = Vec::new(); let mut left_limit = dry_run_limit; for table in tables { - let ret = do_vacuum_drop_table(table, left_limit).await?; + // only operate fuse table + let ret = if let Ok(fuse_table) = FuseTable::try_from_table(table.as_ref()) { + let table_info = table.get_table_info(); + let operator = fuse_table.get_operator_ref(); + do_vacuum_drop_table(table_info, operator, left_limit).await? + } else { + info!( + "ignore table {}, which is not of FUSE engine. Table engine {}", + table.get_table_info().name, + table.engine() + ); + continue; + }; + if let Some(ret) = ret { list_files.extend(ret); if list_files.len() >= dry_run_limit.unwrap() { diff --git a/src/query/ee/tests/it/storages/fuse/operations/vacuum.rs b/src/query/ee/tests/it/storages/fuse/operations/vacuum.rs index bc2eab3b9d31..0089fcf61b4e 100644 --- a/src/query/ee/tests/it/storages/fuse/operations/vacuum.rs +++ b/src/query/ee/tests/it/storages/fuse/operations/vacuum.rs @@ -12,13 +12,27 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::fmt::Debug; + use databend_common_base::base::tokio; use databend_common_exception::Result; +use databend_common_meta_app::schema::TableInfo; +use databend_common_meta_app::schema::TableMeta; +use databend_common_meta_app::storage::StorageParams; use databend_enterprise_query::storages::fuse::do_vacuum_drop_tables; +use databend_enterprise_query::storages::fuse::operations::vacuum_drop_tables::do_vacuum_drop_table; use databend_query::test_kits::*; +use databend_storages_common_table_meta::table::OPT_KEY_DATABASE_ID; +use opendal::raw::Accessor; +use opendal::raw::AccessorInfo; +use opendal::raw::OpStat; +use opendal::raw::RpStat; +use opendal::EntryMode; +use opendal::Metadata; +use opendal::OperatorBuilder; #[tokio::test(flavor = "multi_thread")] -async fn test_fuse_do_vacuum_drop_table() -> Result<()> { +async fn test_fuse_do_vacuum_drop_tables() -> Result<()> { let fixture = TestFixture::setup().await?; fixture @@ -90,3 +104,130 @@ async fn test_fuse_do_vacuum_drop_table() -> Result<()> { Ok(()) } + +mod test_accessor { + use std::sync::atomic::AtomicBool; + use std::sync::atomic::Ordering; + + use opendal::raw::OpDelete; + use opendal::raw::OpList; + use opendal::raw::RpDelete; + use opendal::raw::RpList; + + use super::*; + + // Accessor that throws an error when deleting dir or files. + #[derive(Debug)] + pub(crate) struct AccessorFaultyDeletion { + hit_delete: AtomicBool, + } + + impl AccessorFaultyDeletion { + pub(crate) fn new() -> Self { + AccessorFaultyDeletion { + hit_delete: AtomicBool::new(false), + } + } + + pub(crate) fn hit_delete_operation(&self) -> bool { + self.hit_delete.load(Ordering::Acquire) + } + } + #[async_trait::async_trait] + impl Accessor for AccessorFaultyDeletion { + type Reader = (); + type BlockingReader = (); + type Writer = (); + type BlockingWriter = (); + type Pager = (); + type BlockingPager = (); + + fn info(&self) -> AccessorInfo { + let mut info = AccessorInfo::default(); + let cap = info.full_capability_mut(); + cap.stat = true; + cap.batch = true; + cap.delete = true; + cap.list = true; + cap.list_with_delimiter_slash = true; + info + } + + async fn stat(&self, _path: &str, _args: OpStat) -> opendal::Result { + let stat = RpStat::new(Metadata::new(EntryMode::DIR)); + Ok(stat) + } + + async fn delete(&self, _path: &str, _args: OpDelete) -> opendal::Result { + self.hit_delete.store(true, Ordering::Release); + Err(opendal::Error::new( + opendal::ErrorKind::Unexpected, + "does not matter (delete)", + )) + } + + async fn list(&self, _path: &str, _args: OpList) -> opendal::Result<(RpList, Self::Pager)> { + Ok((RpList::default(), ())) + } + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_fuse_do_vacuum_drop_table_deletion_error() -> Result<()> { + // do_vacuum_drop_table should return Err if file deletion failed + + let mut table_info = TableInfo::default(); + table_info + .meta + .options + .insert(OPT_KEY_DATABASE_ID.to_owned(), "1".to_owned()); + + use test_accessor::AccessorFaultyDeletion; + // Operator with mocked accessor that will fail on `remove_all` + // + // Note that: + // In real case, `Accessor::batch` will be called (instead of Accessor::delete) + // but all that we need here is let Operator::remove_all failed + let faulty_accessor = std::sync::Arc::new(AccessorFaultyDeletion::new()); + let operator = OperatorBuilder::new(faulty_accessor.clone()).finish(); + + let result = do_vacuum_drop_table(&table_info, &operator, None).await; + assert!(result.is_err()); + + // verify that accessor.delete() was called + assert!(faulty_accessor.hit_delete_operation()); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_fuse_do_vacuum_drop_table_external_storage() -> Result<()> { + // do_vacuum_drop_table should return Ok(None) if external storage detected + + let meta = TableMeta { + storage_params: Some(StorageParams::default()), + ..Default::default() + }; + + let table_info = TableInfo { + meta, + ..Default::default() + }; + + // Accessor passed in does NOT matter in this case, `do_vacuum_drop_table` should + // return Ok(None) before accessor is used. + use test_accessor::AccessorFaultyDeletion; + let accessor = std::sync::Arc::new(AccessorFaultyDeletion::new()); + let operator = OperatorBuilder::new(accessor.clone()).finish(); + + let result = do_vacuum_drop_table(&table_info, &operator, None).await; + + // verify that Ok(None) is returned + assert!(result.is_ok()); + assert!(result.unwrap().is_none()); + + // verify that accessor.delete() was NOT called + assert!(!accessor.hit_delete_operation()); + + Ok(()) +} From fc5a64c60694b0691038c6f38a2e35821a1a56c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E7=82=8E=E6=B3=BC?= Date: Mon, 18 Dec 2023 17:18:44 +0800 Subject: [PATCH 08/20] chore: update doc: since 1.2.258 `ttl` is supported by meta-service (#14058) --- src/meta/client/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/meta/client/src/lib.rs b/src/meta/client/src/lib.rs index 628fc51547a4..ee8c65390a44 100644 --- a/src/meta/client/src/lib.rs +++ b/src/meta/client/src/lib.rs @@ -78,7 +78,7 @@ pub static METACLI_COMMIT_SEMVER: LazyLock = LazyLock::new(|| { /// - 2023-10-20: since 1.2.176: /// Meta client: call stream api: kv_read_v1(), revert to 1.1.32 if server < 1.2.163 /// -/// - 2023-12-16: since TODO: +/// - 2023-12-16: since 1.2.258: /// Meta service: add: ttl to TxnPutRequest and Upsert /// /// Server feature set: From ebec8448479b6b45b51cacd1edd38a5fd4183bd3 Mon Sep 17 00:00:00 2001 From: dantengsky Date: Mon, 18 Dec 2023 21:17:19 +0800 Subject: [PATCH 09/20] fix: treat change$* as internal columns (#14061) * fix: treat change$* as internal column * fix: more internel columns - `ORIGIN_BLOCK_ROW_NUM_COL_NAME` - `BASE_ROW_ID_COL_NAME` * add sqllogic test * refine sqllogic test --- src/query/expression/src/schema.rs | 7 +++++ .../ee/01_ee_system/01_0003_stream.test | 31 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/src/query/expression/src/schema.rs b/src/query/expression/src/schema.rs index 8372513718af..e3f5b802f782 100644 --- a/src/query/expression/src/schema.rs +++ b/src/query/expression/src/schema.rs @@ -102,6 +102,13 @@ pub fn is_internal_column(column_name: &str) -> bool { | BASE_BLOCK_IDS_COL_NAME | ROW_NUMBER_COL_NAME | PREDICATE_COLUMN_NAME + | CHANGE_ACTION_COL_NAME + | CHANGE_IS_UPDATE_COL_NAME + | CHANGE_ROW_ID_COL_NAME + // change$row_id might be expended + // to the computation of the two following internal columns + | ORIGIN_BLOCK_ROW_NUM_COL_NAME + | BASE_ROW_ID_COL_NAME ) } diff --git a/tests/sqllogictests/suites/ee/01_ee_system/01_0003_stream.test b/tests/sqllogictests/suites/ee/01_ee_system/01_0003_stream.test index 1406a8919cc4..dcb8b8f53695 100644 --- a/tests/sqllogictests/suites/ee/01_ee_system/01_0003_stream.test +++ b/tests/sqllogictests/suites/ee/01_ee_system/01_0003_stream.test @@ -223,5 +223,36 @@ drop table s2 statement ok drop stream s2 +############### +# issue 14062 # +############### + +statement ok +create table t_14062 (c int); + +statement ok +create stream s_14062 on table t_14062 ; + +statement ok +insert into t_14062 values(1); + +query T +select * from s_14062 where change$action = 'INSERT'; +---- +1 + +query T +select * from s_14062 where change$is_update = false; +---- +1 + +query T +select * from s_14062 where change$row_id = '1'; +---- + +###################### +# end of issue 14062 # +###################### + statement ok DROP DATABASE IF EXISTS test_stream From 614d5561356ce1c44950eee8dda8a4234a4feae4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E7=82=8E=E6=B3=BC?= Date: Mon, 18 Dec 2023 21:17:45 +0800 Subject: [PATCH 10/20] chore: fix license (#14060) --- src/common/arrow/tests/it/main.rs | 2 +- src/common/base/tests/it/fixed_heap.rs | 2 +- src/common/base/tests/it/main.rs | 2 +- src/common/base/tests/it/pool.rs | 2 +- src/common/base/tests/it/progress.rs | 2 +- src/common/base/tests/it/runtime.rs | 2 +- src/common/base/tests/it/stoppable.rs | 2 +- src/common/cache/tests/it/cache.rs | 2 +- src/common/cache/tests/it/cache/lru.rs | 2 +- src/common/cache/tests/it/main.rs | 2 +- src/common/grpc/tests/it/dns_resolver.rs | 2 +- src/common/grpc/tests/it/grpc_token.rs | 2 +- src/common/grpc/tests/it/main.rs | 2 +- src/common/hashtable/tests/it/main.rs | 2 +- src/common/http/tests/it/health.rs | 2 +- src/common/http/tests/it/main.rs | 2 +- src/common/io/tests/it/binary_read.rs | 2 +- src/common/io/tests/it/binary_write.rs | 2 +- src/common/io/tests/it/bincode_serialization.rs | 2 +- src/common/io/tests/it/borsh_serialization.rs | 2 +- src/common/io/tests/it/cursor_ext/mod.rs | 2 +- src/common/io/tests/it/escape.rs | 2 +- src/common/io/tests/it/main.rs | 2 +- src/common/metrics/tests/it/main.rs | 2 +- src/common/storage/tests/it/main.rs | 2 +- src/meta/app/tests/it/user_grant.rs | 2 +- src/meta/app/tests/it/user_info.rs | 2 +- src/meta/app/tests/it/user_privilege.rs | 2 +- src/meta/app/tests/it/user_quota.rs | 2 +- src/meta/client/tests/it/grpc_client.rs | 2 +- src/meta/client/tests/it/grpc_server.rs | 2 +- src/meta/client/tests/it/main.rs | 2 +- src/meta/embedded/tests/it/kv_api_impl.rs | 2 +- src/meta/embedded/tests/it/schema_api_impl.rs | 2 +- src/meta/proto-conv/tests/it/main.rs | 2 +- src/meta/proto-conv/tests/it/proto_conv.rs | 2 +- src/meta/proto-conv/tests/it/user_proto_conv.rs | 2 +- src/meta/proto-conv/tests/it/v037_index_meta.rs | 2 +- src/meta/proto-conv/tests/it/v041_virtual_column.rs | 2 +- src/meta/proto-conv/tests/it/v044_table_meta.rs | 2 +- src/meta/proto-conv/tests/it/v046_index_meta.rs | 2 +- src/meta/proto-conv/tests/it/v047_catalog_meta.rs | 2 +- src/meta/proto-conv/tests/it/v052_hive_catalog_config.rs | 2 +- src/meta/proto-conv/tests/it/v054_index_meta.rs | 2 +- src/meta/protos/proto/index.proto | 2 +- src/meta/protos/proto/lock.proto | 2 +- src/meta/protos/proto/virtual_column.proto | 2 +- src/meta/protos/tests/it/main.rs | 2 +- src/meta/raft-store/tests/it/config.rs | 2 +- src/meta/raft-store/tests/it/log.rs | 2 +- src/meta/raft-store/tests/it/state.rs | 2 +- src/meta/raft-store/tests/it/state_machine/mod.rs | 2 +- src/meta/raft-store/tests/it/state_machine/schema_api_impl.rs | 2 +- src/meta/raft-store/tests/it/testing.rs | 2 +- src/meta/raft-store/tests/it/types.rs | 2 +- src/meta/service/tests/it/api/http/mod.rs | 2 +- src/meta/service/tests/it/api/http_service.rs | 2 +- src/meta/service/tests/it/api/mod.rs | 2 +- src/meta/service/tests/it/configs.rs | 2 +- src/meta/service/tests/it/grpc/metasrv_connection_error.rs | 2 +- src/meta/service/tests/it/grpc/metasrv_grpc_api.rs | 2 +- src/meta/service/tests/it/grpc/metasrv_grpc_export.rs | 2 +- src/meta/service/tests/it/grpc/metasrv_grpc_handshake.rs | 2 +- src/meta/service/tests/it/grpc/metasrv_grpc_kv_api.rs | 2 +- .../tests/it/grpc/metasrv_grpc_kv_api_restart_cluster.rs | 2 +- src/meta/service/tests/it/grpc/metasrv_grpc_kv_read_v1.rs | 2 +- src/meta/service/tests/it/grpc/metasrv_grpc_schema_api.rs | 2 +- .../it/grpc/metasrv_grpc_schema_api_follower_follower.rs | 2 +- .../tests/it/grpc/metasrv_grpc_schema_api_leader_follower.rs | 2 +- src/meta/service/tests/it/grpc/metasrv_grpc_tls.rs | 2 +- src/meta/service/tests/it/grpc/metasrv_grpc_watch.rs | 2 +- src/meta/service/tests/it/grpc/mod.rs | 2 +- src/meta/service/tests/it/main.rs | 2 +- src/meta/service/tests/it/meta_node/meta_node_lifecycle.rs | 2 +- src/meta/service/tests/it/meta_node/mod.rs | 2 +- src/meta/service/tests/it/store.rs | 2 +- src/meta/service/tests/it/testing.rs | 2 +- src/meta/service/tests/it/tests/mod.rs | 2 +- src/meta/service/tests/it/tests/service.rs | 2 +- src/meta/service/tests/it/tests/tls_constants.rs | 2 +- src/meta/sled-store/tests/it/sled_tree.rs | 2 +- src/meta/sled-store/tests/it/sled_txn_tree.rs | 2 +- src/meta/sled-store/tests/it/testing/fake_key_spaces.rs | 2 +- .../sled-store/tests/it/testing/fake_state_machine_meta.rs | 2 +- src/meta/sled-store/tests/it/testing/mod.rs | 2 +- src/meta/types/proto/meta.proto | 2 +- src/meta/types/proto/request.proto | 2 +- src/meta/types/tests/it/cluster.rs | 2 +- src/query/ast/tests/it/main.rs | 2 +- src/query/expression/tests/it/fill_field_default_value.rs | 2 +- src/query/formats/tests/it/output_format_tcsv.rs | 2 +- src/query/formats/tests/it/output_format_utils.rs | 2 +- src/query/functions/tests/it/scalars/comparison.rs | 2 +- src/query/management/tests/it/cluster.rs | 2 +- src/query/management/tests/it/main.rs | 2 +- src/query/management/tests/it/stage.rs | 2 +- src/query/management/tests/it/udf.rs | 2 +- src/query/management/tests/it/user.rs | 2 +- src/query/pipeline/core/tests/it/main.rs | 2 +- src/query/pipeline/core/tests/it/pipelines/mod.rs | 2 +- src/query/service/tests/it/api/http/cluster.rs | 2 +- src/query/service/tests/it/api/http/logs.rs | 2 +- src/query/service/tests/it/api/http/mod.rs | 2 +- src/query/service/tests/it/api/http/status.rs | 2 +- src/query/service/tests/it/api/http_service.rs | 2 +- src/query/service/tests/it/api/mod.rs | 2 +- src/query/service/tests/it/api/rpc_service.rs | 2 +- src/query/service/tests/it/clusters.rs | 2 +- src/query/service/tests/it/configs.rs | 2 +- src/query/service/tests/it/distributed/mod.rs | 2 +- src/query/service/tests/it/interpreters/mod.rs | 2 +- src/query/service/tests/it/interpreters/union.rs | 2 +- src/query/service/tests/it/main.rs | 2 +- src/query/service/tests/it/metrics.rs | 2 +- src/query/service/tests/it/pipelines/mod.rs | 2 +- .../service/tests/it/servers/flight_sql/flight_sql_server.rs | 2 +- .../service/tests/it/servers/http/http_query_handlers.rs | 2 +- src/query/service/tests/it/servers/http/json_block.rs | 2 +- src/query/service/tests/it/servers/http/mod.rs | 2 +- src/query/service/tests/it/servers/mod.rs | 2 +- src/query/service/tests/it/sessions/mod.rs | 2 +- src/query/service/tests/it/sessions/session.rs | 2 +- src/query/service/tests/it/sessions/session_setting.rs | 2 +- src/query/service/tests/it/sql/mod.rs | 2 +- src/query/service/tests/it/storages/mod.rs | 2 +- src/query/service/tests/it/storages/system.rs | 2 +- src/query/service/tests/it/tests/catalog.rs | 2 +- src/query/service/tests/it/tests/mod.rs | 2 +- src/query/service/tests/it/tests/tls_constants.rs | 2 +- src/query/settings/tests/it/main.rs | 2 +- src/query/storages/common/index/tests/it/filters/xor8.rs | 2 +- src/query/storages/hive/hive/tests/it/hive_file_splitter.rs | 2 +- src/query/storages/hive/hive/tests/it/main.rs | 2 +- src/query/storages/parquet/tests/it/main.rs | 2 +- src/query/storages/parquet/tests/it/merge_io.rs | 2 +- src/query/users/tests/it/main.rs | 2 +- src/query/users/tests/it/mod.rs | 4 ++-- src/query/users/tests/it/network_policy.rs | 2 +- src/query/users/tests/it/role_util.rs | 2 +- src/query/users/tests/it/user_mgr.rs | 2 +- src/query/users/tests/it/user_udf.rs | 2 +- tests/sqllogictests/src/arg.rs | 2 +- tests/sqllogictests/src/client/http_client.rs | 2 +- tests/sqllogictests/src/client/mod.rs | 2 +- tests/sqllogictests/src/client/mysql_client.rs | 2 +- tests/sqllogictests/src/error.rs | 2 +- tests/sqllogictests/src/main.rs | 2 +- tests/sqllogictests/src/util.rs | 2 +- 148 files changed, 149 insertions(+), 149 deletions(-) diff --git a/src/common/arrow/tests/it/main.rs b/src/common/arrow/tests/it/main.rs index 47a98a06567c..83c1d406d287 100644 --- a/src/common/arrow/tests/it/main.rs +++ b/src/common/arrow/tests/it/main.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/common/base/tests/it/fixed_heap.rs b/src/common/base/tests/it/fixed_heap.rs index a9e47d148d1f..5f3536e8138a 100644 --- a/src/common/base/tests/it/fixed_heap.rs +++ b/src/common/base/tests/it/fixed_heap.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/common/base/tests/it/main.rs b/src/common/base/tests/it/main.rs index 34da796e0cb4..5f0df1b3c283 100644 --- a/src/common/base/tests/it/main.rs +++ b/src/common/base/tests/it/main.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/common/base/tests/it/pool.rs b/src/common/base/tests/it/pool.rs index 4e12964ac11a..50490cb2972b 100644 --- a/src/common/base/tests/it/pool.rs +++ b/src/common/base/tests/it/pool.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/common/base/tests/it/progress.rs b/src/common/base/tests/it/progress.rs index af9d66222809..95c80bb88de6 100644 --- a/src/common/base/tests/it/progress.rs +++ b/src/common/base/tests/it/progress.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/common/base/tests/it/runtime.rs b/src/common/base/tests/it/runtime.rs index 91cfaaa17ccf..9067e5945d1e 100644 --- a/src/common/base/tests/it/runtime.rs +++ b/src/common/base/tests/it/runtime.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/common/base/tests/it/stoppable.rs b/src/common/base/tests/it/stoppable.rs index be6179c1200e..25d33e6c3cf3 100644 --- a/src/common/base/tests/it/stoppable.rs +++ b/src/common/base/tests/it/stoppable.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/common/cache/tests/it/cache.rs b/src/common/cache/tests/it/cache.rs index ff36463970ea..ac9d154e44ff 100644 --- a/src/common/cache/tests/it/cache.rs +++ b/src/common/cache/tests/it/cache.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/common/cache/tests/it/cache/lru.rs b/src/common/cache/tests/it/cache/lru.rs index 84453349948f..ae9956afd1d7 100644 --- a/src/common/cache/tests/it/cache/lru.rs +++ b/src/common/cache/tests/it/cache/lru.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/common/cache/tests/it/main.rs b/src/common/cache/tests/it/main.rs index 0f9ba048238f..e3ddf0b21724 100644 --- a/src/common/cache/tests/it/main.rs +++ b/src/common/cache/tests/it/main.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/common/grpc/tests/it/dns_resolver.rs b/src/common/grpc/tests/it/dns_resolver.rs index 47c9a2cdb385..078625c56059 100644 --- a/src/common/grpc/tests/it/dns_resolver.rs +++ b/src/common/grpc/tests/it/dns_resolver.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/common/grpc/tests/it/grpc_token.rs b/src/common/grpc/tests/it/grpc_token.rs index 3dc6bb40b2ea..11e18184bd26 100644 --- a/src/common/grpc/tests/it/grpc_token.rs +++ b/src/common/grpc/tests/it/grpc_token.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/common/grpc/tests/it/main.rs b/src/common/grpc/tests/it/main.rs index a70c89f02763..3eb53699d934 100644 --- a/src/common/grpc/tests/it/main.rs +++ b/src/common/grpc/tests/it/main.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/common/hashtable/tests/it/main.rs b/src/common/hashtable/tests/it/main.rs index 8d6f461bbbd7..897587bffdd8 100644 --- a/src/common/hashtable/tests/it/main.rs +++ b/src/common/hashtable/tests/it/main.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/common/http/tests/it/health.rs b/src/common/http/tests/it/health.rs index 8742268304e8..e8a5c9241819 100644 --- a/src/common/http/tests/it/health.rs +++ b/src/common/http/tests/it/health.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/common/http/tests/it/main.rs b/src/common/http/tests/it/main.rs index 6a23fe192ec8..be1339915b5e 100644 --- a/src/common/http/tests/it/main.rs +++ b/src/common/http/tests/it/main.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/common/io/tests/it/binary_read.rs b/src/common/io/tests/it/binary_read.rs index ed66f2b363a6..040fa2ce6196 100644 --- a/src/common/io/tests/it/binary_read.rs +++ b/src/common/io/tests/it/binary_read.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/common/io/tests/it/binary_write.rs b/src/common/io/tests/it/binary_write.rs index 1c10f8e13467..02bb0f8e2405 100644 --- a/src/common/io/tests/it/binary_write.rs +++ b/src/common/io/tests/it/binary_write.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/common/io/tests/it/bincode_serialization.rs b/src/common/io/tests/it/bincode_serialization.rs index 7a47860ad26d..04e5ad4e329e 100644 --- a/src/common/io/tests/it/bincode_serialization.rs +++ b/src/common/io/tests/it/bincode_serialization.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/common/io/tests/it/borsh_serialization.rs b/src/common/io/tests/it/borsh_serialization.rs index f1a43b72f740..37010700561e 100644 --- a/src/common/io/tests/it/borsh_serialization.rs +++ b/src/common/io/tests/it/borsh_serialization.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/common/io/tests/it/cursor_ext/mod.rs b/src/common/io/tests/it/cursor_ext/mod.rs index a27c5b550190..8a2e0c122484 100644 --- a/src/common/io/tests/it/cursor_ext/mod.rs +++ b/src/common/io/tests/it/cursor_ext/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/common/io/tests/it/escape.rs b/src/common/io/tests/it/escape.rs index 3eb122976b45..42bb5552a6d1 100644 --- a/src/common/io/tests/it/escape.rs +++ b/src/common/io/tests/it/escape.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/common/io/tests/it/main.rs b/src/common/io/tests/it/main.rs index 9ef129835d91..9fe0f545ec1b 100644 --- a/src/common/io/tests/it/main.rs +++ b/src/common/io/tests/it/main.rs @@ -1,5 +1,5 @@ #![feature(cursor_remaining)] -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/common/metrics/tests/it/main.rs b/src/common/metrics/tests/it/main.rs index 152f46c1ef47..de0370d388c6 100644 --- a/src/common/metrics/tests/it/main.rs +++ b/src/common/metrics/tests/it/main.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/common/storage/tests/it/main.rs b/src/common/storage/tests/it/main.rs index cee81879f7fa..befc896345aa 100644 --- a/src/common/storage/tests/it/main.rs +++ b/src/common/storage/tests/it/main.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/app/tests/it/user_grant.rs b/src/meta/app/tests/it/user_grant.rs index c133ac072b58..e5d1f60364cf 100644 --- a/src/meta/app/tests/it/user_grant.rs +++ b/src/meta/app/tests/it/user_grant.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/app/tests/it/user_info.rs b/src/meta/app/tests/it/user_info.rs index 28da573b9dae..287e8fd8ca85 100644 --- a/src/meta/app/tests/it/user_info.rs +++ b/src/meta/app/tests/it/user_info.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/app/tests/it/user_privilege.rs b/src/meta/app/tests/it/user_privilege.rs index 0085ca59110e..0966a669b4a0 100644 --- a/src/meta/app/tests/it/user_privilege.rs +++ b/src/meta/app/tests/it/user_privilege.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/app/tests/it/user_quota.rs b/src/meta/app/tests/it/user_quota.rs index 83e54a795754..dfa6c045c7cb 100644 --- a/src/meta/app/tests/it/user_quota.rs +++ b/src/meta/app/tests/it/user_quota.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/client/tests/it/grpc_client.rs b/src/meta/client/tests/it/grpc_client.rs index 1892a8e26abe..3faab69b6880 100644 --- a/src/meta/client/tests/it/grpc_client.rs +++ b/src/meta/client/tests/it/grpc_client.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/client/tests/it/grpc_server.rs b/src/meta/client/tests/it/grpc_server.rs index 978dfeec4e85..380d313fe6d1 100644 --- a/src/meta/client/tests/it/grpc_server.rs +++ b/src/meta/client/tests/it/grpc_server.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/client/tests/it/main.rs b/src/meta/client/tests/it/main.rs index c9240fc5616e..72f514b886c9 100644 --- a/src/meta/client/tests/it/main.rs +++ b/src/meta/client/tests/it/main.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/embedded/tests/it/kv_api_impl.rs b/src/meta/embedded/tests/it/kv_api_impl.rs index 9e69a02ab0b6..32806e2ae458 100644 --- a/src/meta/embedded/tests/it/kv_api_impl.rs +++ b/src/meta/embedded/tests/it/kv_api_impl.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/embedded/tests/it/schema_api_impl.rs b/src/meta/embedded/tests/it/schema_api_impl.rs index b699188d22ae..452d24546a9c 100644 --- a/src/meta/embedded/tests/it/schema_api_impl.rs +++ b/src/meta/embedded/tests/it/schema_api_impl.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/proto-conv/tests/it/main.rs b/src/meta/proto-conv/tests/it/main.rs index 85990b0026a4..3bdd6e8eafa8 100644 --- a/src/meta/proto-conv/tests/it/main.rs +++ b/src/meta/proto-conv/tests/it/main.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/proto-conv/tests/it/proto_conv.rs b/src/meta/proto-conv/tests/it/proto_conv.rs index f2916f13aefe..d9f82e6695a2 100644 --- a/src/meta/proto-conv/tests/it/proto_conv.rs +++ b/src/meta/proto-conv/tests/it/proto_conv.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/proto-conv/tests/it/user_proto_conv.rs b/src/meta/proto-conv/tests/it/user_proto_conv.rs index 9d2a151bb795..856f9ff5d1f5 100644 --- a/src/meta/proto-conv/tests/it/user_proto_conv.rs +++ b/src/meta/proto-conv/tests/it/user_proto_conv.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/proto-conv/tests/it/v037_index_meta.rs b/src/meta/proto-conv/tests/it/v037_index_meta.rs index c98030bc8e2d..984ff23489d7 100644 --- a/src/meta/proto-conv/tests/it/v037_index_meta.rs +++ b/src/meta/proto-conv/tests/it/v037_index_meta.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/proto-conv/tests/it/v041_virtual_column.rs b/src/meta/proto-conv/tests/it/v041_virtual_column.rs index 174133133269..314bb6cdd94c 100644 --- a/src/meta/proto-conv/tests/it/v041_virtual_column.rs +++ b/src/meta/proto-conv/tests/it/v041_virtual_column.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/proto-conv/tests/it/v044_table_meta.rs b/src/meta/proto-conv/tests/it/v044_table_meta.rs index 7b13f5751611..5d21bf64542a 100644 --- a/src/meta/proto-conv/tests/it/v044_table_meta.rs +++ b/src/meta/proto-conv/tests/it/v044_table_meta.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/proto-conv/tests/it/v046_index_meta.rs b/src/meta/proto-conv/tests/it/v046_index_meta.rs index 0249f695db1b..ed6ba10110b8 100644 --- a/src/meta/proto-conv/tests/it/v046_index_meta.rs +++ b/src/meta/proto-conv/tests/it/v046_index_meta.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/proto-conv/tests/it/v047_catalog_meta.rs b/src/meta/proto-conv/tests/it/v047_catalog_meta.rs index 1395c3b31b7a..56ada7bc2425 100644 --- a/src/meta/proto-conv/tests/it/v047_catalog_meta.rs +++ b/src/meta/proto-conv/tests/it/v047_catalog_meta.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/proto-conv/tests/it/v052_hive_catalog_config.rs b/src/meta/proto-conv/tests/it/v052_hive_catalog_config.rs index 112f22355e9b..4cc6947937f6 100644 --- a/src/meta/proto-conv/tests/it/v052_hive_catalog_config.rs +++ b/src/meta/proto-conv/tests/it/v052_hive_catalog_config.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/proto-conv/tests/it/v054_index_meta.rs b/src/meta/proto-conv/tests/it/v054_index_meta.rs index dcb7bbc70f45..95ffcd1d7189 100644 --- a/src/meta/proto-conv/tests/it/v054_index_meta.rs +++ b/src/meta/proto-conv/tests/it/v054_index_meta.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/protos/proto/index.proto b/src/meta/protos/proto/index.proto index cb3fb0c90b99..ca5228bb12cd 100644 --- a/src/meta/protos/proto/index.proto +++ b/src/meta/protos/proto/index.proto @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/protos/proto/lock.proto b/src/meta/protos/proto/lock.proto index a79390ab3612..58c93339dcc1 100644 --- a/src/meta/protos/proto/lock.proto +++ b/src/meta/protos/proto/lock.proto @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/protos/proto/virtual_column.proto b/src/meta/protos/proto/virtual_column.proto index ad89a2960f6c..32407d53eab9 100644 --- a/src/meta/protos/proto/virtual_column.proto +++ b/src/meta/protos/proto/virtual_column.proto @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/protos/tests/it/main.rs b/src/meta/protos/tests/it/main.rs index 598edffcf3f6..4ca041965967 100644 --- a/src/meta/protos/tests/it/main.rs +++ b/src/meta/protos/tests/it/main.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/raft-store/tests/it/config.rs b/src/meta/raft-store/tests/it/config.rs index 05eb8a9e7f98..c424c91e360e 100644 --- a/src/meta/raft-store/tests/it/config.rs +++ b/src/meta/raft-store/tests/it/config.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/raft-store/tests/it/log.rs b/src/meta/raft-store/tests/it/log.rs index a221ad65ab3d..0cfb4fbd0345 100644 --- a/src/meta/raft-store/tests/it/log.rs +++ b/src/meta/raft-store/tests/it/log.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/raft-store/tests/it/state.rs b/src/meta/raft-store/tests/it/state.rs index 3128f62f5051..39efcf706f36 100644 --- a/src/meta/raft-store/tests/it/state.rs +++ b/src/meta/raft-store/tests/it/state.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/raft-store/tests/it/state_machine/mod.rs b/src/meta/raft-store/tests/it/state_machine/mod.rs index 2e054343c0eb..41b569af60df 100644 --- a/src/meta/raft-store/tests/it/state_machine/mod.rs +++ b/src/meta/raft-store/tests/it/state_machine/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/raft-store/tests/it/state_machine/schema_api_impl.rs b/src/meta/raft-store/tests/it/state_machine/schema_api_impl.rs index 2616da905302..182102685596 100644 --- a/src/meta/raft-store/tests/it/state_machine/schema_api_impl.rs +++ b/src/meta/raft-store/tests/it/state_machine/schema_api_impl.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/raft-store/tests/it/testing.rs b/src/meta/raft-store/tests/it/testing.rs index 9aeaccad9c78..038951181936 100644 --- a/src/meta/raft-store/tests/it/testing.rs +++ b/src/meta/raft-store/tests/it/testing.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/raft-store/tests/it/types.rs b/src/meta/raft-store/tests/it/types.rs index 8cb8e142c2d4..6d0dfb6ff701 100644 --- a/src/meta/raft-store/tests/it/types.rs +++ b/src/meta/raft-store/tests/it/types.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/service/tests/it/api/http/mod.rs b/src/meta/service/tests/it/api/http/mod.rs index f676f46d463a..a93f18165c8d 100644 --- a/src/meta/service/tests/it/api/http/mod.rs +++ b/src/meta/service/tests/it/api/http/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/service/tests/it/api/http_service.rs b/src/meta/service/tests/it/api/http_service.rs index 4c482faaf772..8cf08014941f 100644 --- a/src/meta/service/tests/it/api/http_service.rs +++ b/src/meta/service/tests/it/api/http_service.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/service/tests/it/api/mod.rs b/src/meta/service/tests/it/api/mod.rs index 46c7f0efae5e..06be9cdd30d4 100644 --- a/src/meta/service/tests/it/api/mod.rs +++ b/src/meta/service/tests/it/api/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/service/tests/it/configs.rs b/src/meta/service/tests/it/configs.rs index 8403dcac7613..19c18203ef27 100644 --- a/src/meta/service/tests/it/configs.rs +++ b/src/meta/service/tests/it/configs.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/service/tests/it/grpc/metasrv_connection_error.rs b/src/meta/service/tests/it/grpc/metasrv_connection_error.rs index bae768343b97..61a7632d0ef6 100644 --- a/src/meta/service/tests/it/grpc/metasrv_connection_error.rs +++ b/src/meta/service/tests/it/grpc/metasrv_connection_error.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_api.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_api.rs index d7a4d3dfd13a..17561ae0f3bb 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_api.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_api.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs index 58d1f9bbfa12..474e56a2657e 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_handshake.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_handshake.rs index 5f006daf3e2e..9a154470d540 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_handshake.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_handshake.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_kv_api.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_kv_api.rs index a17d2b0347ee..0b51b6922e31 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_kv_api.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_kv_api.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_kv_api_restart_cluster.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_kv_api_restart_cluster.rs index d1b55091eb1c..999991182c04 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_kv_api_restart_cluster.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_kv_api_restart_cluster.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_kv_read_v1.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_kv_read_v1.rs index 2cc106df3e2e..d4dd239b143d 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_kv_read_v1.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_kv_read_v1.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_schema_api.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_schema_api.rs index 693ff5c9b5fb..f0bfe26db237 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_schema_api.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_schema_api.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_schema_api_follower_follower.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_schema_api_follower_follower.rs index d505cef152d6..cd3e1ec8fb15 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_schema_api_follower_follower.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_schema_api_follower_follower.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_schema_api_leader_follower.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_schema_api_leader_follower.rs index 9fb32feed888..9ef4319638b9 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_schema_api_leader_follower.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_schema_api_leader_follower.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_tls.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_tls.rs index 5b17b8704d7a..fa30fd404a24 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_tls.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_tls.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_watch.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_watch.rs index e8b4c6009bb3..871546115df4 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_watch.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_watch.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/service/tests/it/grpc/mod.rs b/src/meta/service/tests/it/grpc/mod.rs index 8984498127b0..55692e705c3d 100644 --- a/src/meta/service/tests/it/grpc/mod.rs +++ b/src/meta/service/tests/it/grpc/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/service/tests/it/main.rs b/src/meta/service/tests/it/main.rs index a755dd2617a9..fe25f6346e52 100644 --- a/src/meta/service/tests/it/main.rs +++ b/src/meta/service/tests/it/main.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/service/tests/it/meta_node/meta_node_lifecycle.rs b/src/meta/service/tests/it/meta_node/meta_node_lifecycle.rs index b74fd70e79ea..f50406139a32 100644 --- a/src/meta/service/tests/it/meta_node/meta_node_lifecycle.rs +++ b/src/meta/service/tests/it/meta_node/meta_node_lifecycle.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/service/tests/it/meta_node/mod.rs b/src/meta/service/tests/it/meta_node/mod.rs index 7e3fe788e2e5..0469da7482fc 100644 --- a/src/meta/service/tests/it/meta_node/mod.rs +++ b/src/meta/service/tests/it/meta_node/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/service/tests/it/store.rs b/src/meta/service/tests/it/store.rs index d958a41ec1b8..0be7168d1ebe 100644 --- a/src/meta/service/tests/it/store.rs +++ b/src/meta/service/tests/it/store.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/service/tests/it/testing.rs b/src/meta/service/tests/it/testing.rs index 05af62b21c89..39437c67cbd6 100644 --- a/src/meta/service/tests/it/testing.rs +++ b/src/meta/service/tests/it/testing.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/service/tests/it/tests/mod.rs b/src/meta/service/tests/it/tests/mod.rs index 691b1a6c12a2..0703956184fa 100644 --- a/src/meta/service/tests/it/tests/mod.rs +++ b/src/meta/service/tests/it/tests/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/service/tests/it/tests/service.rs b/src/meta/service/tests/it/tests/service.rs index 025f2e667e54..14689da1542e 100644 --- a/src/meta/service/tests/it/tests/service.rs +++ b/src/meta/service/tests/it/tests/service.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/service/tests/it/tests/tls_constants.rs b/src/meta/service/tests/it/tests/tls_constants.rs index 86cb480f6d6d..f9a6bcd54588 100644 --- a/src/meta/service/tests/it/tests/tls_constants.rs +++ b/src/meta/service/tests/it/tests/tls_constants.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/sled-store/tests/it/sled_tree.rs b/src/meta/sled-store/tests/it/sled_tree.rs index e42af93d3a8e..7141d0b2bc93 100644 --- a/src/meta/sled-store/tests/it/sled_tree.rs +++ b/src/meta/sled-store/tests/it/sled_tree.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/sled-store/tests/it/sled_txn_tree.rs b/src/meta/sled-store/tests/it/sled_txn_tree.rs index 08d1991d2aa1..e3b9b1920a83 100644 --- a/src/meta/sled-store/tests/it/sled_txn_tree.rs +++ b/src/meta/sled-store/tests/it/sled_txn_tree.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/sled-store/tests/it/testing/fake_key_spaces.rs b/src/meta/sled-store/tests/it/testing/fake_key_spaces.rs index 9b2f7ba9789d..529c40ebc619 100644 --- a/src/meta/sled-store/tests/it/testing/fake_key_spaces.rs +++ b/src/meta/sled-store/tests/it/testing/fake_key_spaces.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/sled-store/tests/it/testing/fake_state_machine_meta.rs b/src/meta/sled-store/tests/it/testing/fake_state_machine_meta.rs index 57d0571da4ee..d2f3b2330409 100644 --- a/src/meta/sled-store/tests/it/testing/fake_state_machine_meta.rs +++ b/src/meta/sled-store/tests/it/testing/fake_state_machine_meta.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/sled-store/tests/it/testing/mod.rs b/src/meta/sled-store/tests/it/testing/mod.rs index 0aedf9246e4b..3e4daa66f889 100644 --- a/src/meta/sled-store/tests/it/testing/mod.rs +++ b/src/meta/sled-store/tests/it/testing/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/types/proto/meta.proto b/src/meta/types/proto/meta.proto index 90be9481d16a..1f5db90387c7 100644 --- a/src/meta/types/proto/meta.proto +++ b/src/meta/types/proto/meta.proto @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/types/proto/request.proto b/src/meta/types/proto/request.proto index e2b21862dbcd..41f76d64a759 100644 --- a/src/meta/types/proto/request.proto +++ b/src/meta/types/proto/request.proto @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/meta/types/tests/it/cluster.rs b/src/meta/types/tests/it/cluster.rs index b829c5b19a08..8ba2e3b62867 100644 --- a/src/meta/types/tests/it/cluster.rs +++ b/src/meta/types/tests/it/cluster.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/ast/tests/it/main.rs b/src/query/ast/tests/it/main.rs index 32f7b0515470..9565dfc9d3ea 100644 --- a/src/query/ast/tests/it/main.rs +++ b/src/query/ast/tests/it/main.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/expression/tests/it/fill_field_default_value.rs b/src/query/expression/tests/it/fill_field_default_value.rs index 55c750c8d806..aa97039257c6 100644 --- a/src/query/expression/tests/it/fill_field_default_value.rs +++ b/src/query/expression/tests/it/fill_field_default_value.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/formats/tests/it/output_format_tcsv.rs b/src/query/formats/tests/it/output_format_tcsv.rs index a49936898af8..41f6b2cc9cd6 100644 --- a/src/query/formats/tests/it/output_format_tcsv.rs +++ b/src/query/formats/tests/it/output_format_tcsv.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/formats/tests/it/output_format_utils.rs b/src/query/formats/tests/it/output_format_utils.rs index a735aa7d6063..4c6b0fc43dd4 100644 --- a/src/query/formats/tests/it/output_format_utils.rs +++ b/src/query/formats/tests/it/output_format_utils.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/functions/tests/it/scalars/comparison.rs b/src/query/functions/tests/it/scalars/comparison.rs index 730c968c6b72..7a3f8a7c4564 100644 --- a/src/query/functions/tests/it/scalars/comparison.rs +++ b/src/query/functions/tests/it/scalars/comparison.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/management/tests/it/cluster.rs b/src/query/management/tests/it/cluster.rs index 2f58c8a2c3a5..aacffb4a4d4a 100644 --- a/src/query/management/tests/it/cluster.rs +++ b/src/query/management/tests/it/cluster.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/management/tests/it/main.rs b/src/query/management/tests/it/main.rs index 50bef4be9db6..44a778974380 100644 --- a/src/query/management/tests/it/main.rs +++ b/src/query/management/tests/it/main.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/management/tests/it/stage.rs b/src/query/management/tests/it/stage.rs index 44ba0e1437ee..86f4c0da35b1 100644 --- a/src/query/management/tests/it/stage.rs +++ b/src/query/management/tests/it/stage.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/management/tests/it/udf.rs b/src/query/management/tests/it/udf.rs index d685aecbe011..5adafd95d0f1 100644 --- a/src/query/management/tests/it/udf.rs +++ b/src/query/management/tests/it/udf.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/management/tests/it/user.rs b/src/query/management/tests/it/user.rs index 9d448800a276..869f992291d4 100644 --- a/src/query/management/tests/it/user.rs +++ b/src/query/management/tests/it/user.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/pipeline/core/tests/it/main.rs b/src/query/pipeline/core/tests/it/main.rs index f3a88f12ffad..1d3dd8d9e131 100644 --- a/src/query/pipeline/core/tests/it/main.rs +++ b/src/query/pipeline/core/tests/it/main.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/pipeline/core/tests/it/pipelines/mod.rs b/src/query/pipeline/core/tests/it/pipelines/mod.rs index 128562d08891..b8457fa84088 100644 --- a/src/query/pipeline/core/tests/it/pipelines/mod.rs +++ b/src/query/pipeline/core/tests/it/pipelines/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/api/http/cluster.rs b/src/query/service/tests/it/api/http/cluster.rs index 39c7edba60d8..f1fc5f355665 100644 --- a/src/query/service/tests/it/api/http/cluster.rs +++ b/src/query/service/tests/it/api/http/cluster.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/api/http/logs.rs b/src/query/service/tests/it/api/http/logs.rs index 95f6ee8d9f70..ad1e3759fa45 100644 --- a/src/query/service/tests/it/api/http/logs.rs +++ b/src/query/service/tests/it/api/http/logs.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/api/http/mod.rs b/src/query/service/tests/it/api/http/mod.rs index de6ab219ecaa..54e2d417b9ab 100644 --- a/src/query/service/tests/it/api/http/mod.rs +++ b/src/query/service/tests/it/api/http/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/api/http/status.rs b/src/query/service/tests/it/api/http/status.rs index 7fec056ef6dd..b5b5db2d3ff2 100644 --- a/src/query/service/tests/it/api/http/status.rs +++ b/src/query/service/tests/it/api/http/status.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/api/http_service.rs b/src/query/service/tests/it/api/http_service.rs index c85f7e42e30c..8e4ee445bf39 100644 --- a/src/query/service/tests/it/api/http_service.rs +++ b/src/query/service/tests/it/api/http_service.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/api/mod.rs b/src/query/service/tests/it/api/mod.rs index 8be25df45d09..0dc75928aaa5 100644 --- a/src/query/service/tests/it/api/mod.rs +++ b/src/query/service/tests/it/api/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/api/rpc_service.rs b/src/query/service/tests/it/api/rpc_service.rs index cd8f95b02228..f7b9a0018bae 100644 --- a/src/query/service/tests/it/api/rpc_service.rs +++ b/src/query/service/tests/it/api/rpc_service.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/clusters.rs b/src/query/service/tests/it/clusters.rs index e9242b7a747d..8bb4ca7e1425 100644 --- a/src/query/service/tests/it/clusters.rs +++ b/src/query/service/tests/it/clusters.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/configs.rs b/src/query/service/tests/it/configs.rs index f91df5f9f4f5..7318f861aaad 100644 --- a/src/query/service/tests/it/configs.rs +++ b/src/query/service/tests/it/configs.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/distributed/mod.rs b/src/query/service/tests/it/distributed/mod.rs index 6ed56a78dfd8..dff132cb3a8a 100644 --- a/src/query/service/tests/it/distributed/mod.rs +++ b/src/query/service/tests/it/distributed/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/interpreters/mod.rs b/src/query/service/tests/it/interpreters/mod.rs index 5dcc67376db7..dc94aeb01fff 100644 --- a/src/query/service/tests/it/interpreters/mod.rs +++ b/src/query/service/tests/it/interpreters/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/interpreters/union.rs b/src/query/service/tests/it/interpreters/union.rs index 24fb475a3aaa..f5f42b168c2a 100644 --- a/src/query/service/tests/it/interpreters/union.rs +++ b/src/query/service/tests/it/interpreters/union.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/main.rs b/src/query/service/tests/it/main.rs index bfdad79af04c..d87af79b7c01 100644 --- a/src/query/service/tests/it/main.rs +++ b/src/query/service/tests/it/main.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/metrics.rs b/src/query/service/tests/it/metrics.rs index f2ecb8a798b1..7b9291b1d77a 100644 --- a/src/query/service/tests/it/metrics.rs +++ b/src/query/service/tests/it/metrics.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/pipelines/mod.rs b/src/query/service/tests/it/pipelines/mod.rs index 82232659ef94..b5bc5a74e475 100644 --- a/src/query/service/tests/it/pipelines/mod.rs +++ b/src/query/service/tests/it/pipelines/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/servers/flight_sql/flight_sql_server.rs b/src/query/service/tests/it/servers/flight_sql/flight_sql_server.rs index 44612a7b443b..6173f412da36 100644 --- a/src/query/service/tests/it/servers/flight_sql/flight_sql_server.rs +++ b/src/query/service/tests/it/servers/flight_sql/flight_sql_server.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/servers/http/http_query_handlers.rs b/src/query/service/tests/it/servers/http/http_query_handlers.rs index d2db0f123046..2a8b05da1c92 100644 --- a/src/query/service/tests/it/servers/http/http_query_handlers.rs +++ b/src/query/service/tests/it/servers/http/http_query_handlers.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/servers/http/json_block.rs b/src/query/service/tests/it/servers/http/json_block.rs index 9e0858873893..e855a8aeb25f 100644 --- a/src/query/service/tests/it/servers/http/json_block.rs +++ b/src/query/service/tests/it/servers/http/json_block.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/servers/http/mod.rs b/src/query/service/tests/it/servers/http/mod.rs index 7af083933e5d..33de44308ec6 100644 --- a/src/query/service/tests/it/servers/http/mod.rs +++ b/src/query/service/tests/it/servers/http/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/servers/mod.rs b/src/query/service/tests/it/servers/mod.rs index 74339ff35cd8..5541b9c7536f 100644 --- a/src/query/service/tests/it/servers/mod.rs +++ b/src/query/service/tests/it/servers/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/sessions/mod.rs b/src/query/service/tests/it/sessions/mod.rs index 74ed817c4cdc..b53dd14deca9 100644 --- a/src/query/service/tests/it/sessions/mod.rs +++ b/src/query/service/tests/it/sessions/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/sessions/session.rs b/src/query/service/tests/it/sessions/session.rs index 5ac30b5b1dec..e67ac0baf98b 100644 --- a/src/query/service/tests/it/sessions/session.rs +++ b/src/query/service/tests/it/sessions/session.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/sessions/session_setting.rs b/src/query/service/tests/it/sessions/session_setting.rs index db70998c797c..96ff582ed7a4 100644 --- a/src/query/service/tests/it/sessions/session_setting.rs +++ b/src/query/service/tests/it/sessions/session_setting.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/sql/mod.rs b/src/query/service/tests/it/sql/mod.rs index 560e3ef26068..6e3bdf945074 100644 --- a/src/query/service/tests/it/sql/mod.rs +++ b/src/query/service/tests/it/sql/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/storages/mod.rs b/src/query/service/tests/it/storages/mod.rs index 32442574f0bb..1086efea7dd1 100644 --- a/src/query/service/tests/it/storages/mod.rs +++ b/src/query/service/tests/it/storages/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/storages/system.rs b/src/query/service/tests/it/storages/system.rs index 3bd176d0d7e7..6f9b0c909dca 100644 --- a/src/query/service/tests/it/storages/system.rs +++ b/src/query/service/tests/it/storages/system.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/tests/catalog.rs b/src/query/service/tests/it/tests/catalog.rs index 06dcdbe85eb6..4cf83524348b 100644 --- a/src/query/service/tests/it/tests/catalog.rs +++ b/src/query/service/tests/it/tests/catalog.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/tests/mod.rs b/src/query/service/tests/it/tests/mod.rs index 0ca0e832d0e1..cc437ec230e1 100644 --- a/src/query/service/tests/it/tests/mod.rs +++ b/src/query/service/tests/it/tests/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/service/tests/it/tests/tls_constants.rs b/src/query/service/tests/it/tests/tls_constants.rs index a306027f7141..40a0b85c643b 100644 --- a/src/query/service/tests/it/tests/tls_constants.rs +++ b/src/query/service/tests/it/tests/tls_constants.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/settings/tests/it/main.rs b/src/query/settings/tests/it/main.rs index c6ec4929b099..d31368b4c7e3 100644 --- a/src/query/settings/tests/it/main.rs +++ b/src/query/settings/tests/it/main.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/storages/common/index/tests/it/filters/xor8.rs b/src/query/storages/common/index/tests/it/filters/xor8.rs index d9774b52d905..7a874833f452 100644 --- a/src/query/storages/common/index/tests/it/filters/xor8.rs +++ b/src/query/storages/common/index/tests/it/filters/xor8.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/storages/hive/hive/tests/it/hive_file_splitter.rs b/src/query/storages/hive/hive/tests/it/hive_file_splitter.rs index e1363039aaff..cf77bcf40614 100644 --- a/src/query/storages/hive/hive/tests/it/hive_file_splitter.rs +++ b/src/query/storages/hive/hive/tests/it/hive_file_splitter.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/storages/hive/hive/tests/it/main.rs b/src/query/storages/hive/hive/tests/it/main.rs index 820e5c5d9ed4..78f1d926806f 100644 --- a/src/query/storages/hive/hive/tests/it/main.rs +++ b/src/query/storages/hive/hive/tests/it/main.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/storages/parquet/tests/it/main.rs b/src/query/storages/parquet/tests/it/main.rs index d7d81a7f1655..fc3a852d1f51 100644 --- a/src/query/storages/parquet/tests/it/main.rs +++ b/src/query/storages/parquet/tests/it/main.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/storages/parquet/tests/it/merge_io.rs b/src/query/storages/parquet/tests/it/merge_io.rs index 209ae5447b1f..b4bda7900b31 100644 --- a/src/query/storages/parquet/tests/it/merge_io.rs +++ b/src/query/storages/parquet/tests/it/merge_io.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/users/tests/it/main.rs b/src/query/users/tests/it/main.rs index da88bc961f50..db79f359c4e4 100644 --- a/src/query/users/tests/it/main.rs +++ b/src/query/users/tests/it/main.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/users/tests/it/mod.rs b/src/query/users/tests/it/mod.rs index 7caa6d969b7c..3050ad50e21f 100644 --- a/src/query/users/tests/it/mod.rs +++ b/src/query/users/tests/it/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +mod jwt; mod network_policy; mod role_cache_mgr; mod role_mgr; mod user_mgr; mod user_udf; -mod jwt; diff --git a/src/query/users/tests/it/network_policy.rs b/src/query/users/tests/it/network_policy.rs index 346dcc722731..e1879ab70533 100644 --- a/src/query/users/tests/it/network_policy.rs +++ b/src/query/users/tests/it/network_policy.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/users/tests/it/role_util.rs b/src/query/users/tests/it/role_util.rs index 3faaaf2e65b1..b9124c40b2c2 100644 --- a/src/query/users/tests/it/role_util.rs +++ b/src/query/users/tests/it/role_util.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/users/tests/it/user_mgr.rs b/src/query/users/tests/it/user_mgr.rs index ca63a187a08a..46adbced2ac7 100644 --- a/src/query/users/tests/it/user_mgr.rs +++ b/src/query/users/tests/it/user_mgr.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/query/users/tests/it/user_udf.rs b/src/query/users/tests/it/user_udf.rs index efca38c1a987..62158c7b5bae 100644 --- a/src/query/users/tests/it/user_udf.rs +++ b/src/query/users/tests/it/user_udf.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/tests/sqllogictests/src/arg.rs b/tests/sqllogictests/src/arg.rs index f4546368c051..d6534a6336bd 100644 --- a/tests/sqllogictests/src/arg.rs +++ b/tests/sqllogictests/src/arg.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/tests/sqllogictests/src/client/http_client.rs b/tests/sqllogictests/src/client/http_client.rs index 3256cb0d7657..4c6857615440 100644 --- a/tests/sqllogictests/src/client/http_client.rs +++ b/tests/sqllogictests/src/client/http_client.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/tests/sqllogictests/src/client/mod.rs b/tests/sqllogictests/src/client/mod.rs index d25522d1ed28..439bfb5e3ddc 100644 --- a/tests/sqllogictests/src/client/mod.rs +++ b/tests/sqllogictests/src/client/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/tests/sqllogictests/src/client/mysql_client.rs b/tests/sqllogictests/src/client/mysql_client.rs index 8bfc6bb1d136..b821e760f320 100644 --- a/tests/sqllogictests/src/client/mysql_client.rs +++ b/tests/sqllogictests/src/client/mysql_client.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/tests/sqllogictests/src/error.rs b/tests/sqllogictests/src/error.rs index 3e9fa5e3a45a..1ef95b60062b 100644 --- a/tests/sqllogictests/src/error.rs +++ b/tests/sqllogictests/src/error.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/tests/sqllogictests/src/main.rs b/tests/sqllogictests/src/main.rs index 7ee2eb48095c..d85af5cf7e2c 100644 --- a/tests/sqllogictests/src/main.rs +++ b/tests/sqllogictests/src/main.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/tests/sqllogictests/src/util.rs b/tests/sqllogictests/src/util.rs index 3fc0b6eaed05..df1bd2f1f338 100644 --- a/tests/sqllogictests/src/util.rs +++ b/tests/sqllogictests/src/util.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Datafuse Labs. +// Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. From 39eca439f7164d3938f925717fa531ba88f3d64c Mon Sep 17 00:00:00 2001 From: codedump Date: Mon, 18 Dec 2023 21:24:15 +0800 Subject: [PATCH 11/20] fix: fix get db without DbIdList key (#14059) --- src/meta/api/src/schema_api_impl.rs | 122 +++++++++++++++++++++- src/meta/api/src/schema_api_test_suite.rs | 117 +++++++++++++++++++++ 2 files changed, 238 insertions(+), 1 deletion(-) diff --git a/src/meta/api/src/schema_api_impl.rs b/src/meta/api/src/schema_api_impl.rs index 12304837afc4..93ed01d4ef2a 100644 --- a/src/meta/api/src/schema_api_impl.rs +++ b/src/meta/api/src/schema_api_impl.rs @@ -14,6 +14,8 @@ use std::cmp::min; use std::collections::BTreeMap; +use std::collections::HashMap; +use std::collections::HashSet; use std::fmt::Display; use std::sync::Arc; @@ -485,6 +487,28 @@ impl + ?Sized> SchemaApi for KV { if_then.push(txn_op_put(&db_id_key, serialize_struct(&db_meta)?)); // (db_id) -> db_meta } + + // add DbIdListKey if not exists + let dbid_idlist = DbIdListKey { + tenant: tenant_dbname.tenant.clone(), + db_name: tenant_dbname.db_name.clone(), + }; + let (db_id_list_seq, db_id_list_opt): (_, Option) = + get_pb_value(self, &dbid_idlist).await?; + + if db_id_list_seq == 0 || db_id_list_opt.is_none() { + warn!( + "drop db:{:?}, db_id:{:?} has no DbIdListKey", + tenant_dbname, db_id + ); + + let mut db_id_list = DbIdList::new(); + db_id_list.append(db_id); + + condition.push(txn_cond_seq(&dbid_idlist, Eq, db_id_list_seq)); + // _fd_db_id_list// -> db_id_list + if_then.push(txn_op_put(&dbid_idlist, serialize_struct(&db_id_list)?)); + }; } let txn_req = TxnRequest { @@ -795,7 +819,7 @@ impl + ?Sized> SchemaApi for KV { // List tables by tenant, db_id, table_name. let dbid_tbname_idlist = DbIdListKey { - tenant: req.tenant, + tenant: req.tenant.clone(), // Using a empty db to to list all db_name: "".to_string(), }; @@ -868,6 +892,77 @@ impl + ?Sized> SchemaApi for KV { } } + // `list_database` can list db which has no `DbIdListKey` + if include_drop_db { + // if `include_drop_db` is true, return all db info which not exist in db_info_list + let db_id_set: HashSet = db_info_list + .iter() + .map(|db_info| db_info.ident.db_id) + .collect(); + + let all_dbs = self.list_databases(req).await?; + for db_info in all_dbs { + if !db_id_set.contains(&db_info.ident.db_id) { + warn!( + "get db history db:{:?}, db_id:{:?} has no DbIdListKey", + db_info.name_ident, db_info.ident.db_id + ); + db_info_list.push(db_info); + } + } + } else { + // if `include_drop_db` is false, filter out db which drop_on time out of retention time + let db_id_set: HashSet = db_info_list + .iter() + .map(|db_info| db_info.ident.db_id) + .collect(); + + let all_dbs = self.list_databases(req).await?; + let mut add_dbinfo_map = HashMap::new(); + let mut db_id_list = Vec::new(); + for db_info in all_dbs { + if !db_id_set.contains(&db_info.ident.db_id) { + warn!( + "get db history db:{:?}, db_id:{:?} has no DbIdListKey", + db_info.name_ident, db_info.ident.db_id + ); + db_id_list.push(DatabaseId { + db_id: db_info.ident.db_id, + }); + add_dbinfo_map.insert(db_info.ident.db_id, db_info); + } + } + let inner_keys: Vec = db_id_list + .iter() + .map(|db_id| db_id.to_string_key()) + .collect(); + let mut db_id_list_iter = db_id_list.into_iter(); + for c in inner_keys.chunks(DEFAULT_MGET_SIZE) { + let db_meta_seq_meta_vec: Vec<(u64, Option)> = + mget_pb_values(self, c).await?; + + for (db_meta_seq, db_meta) in db_meta_seq_meta_vec { + let db_id = db_id_list_iter.next().unwrap().db_id; + if db_meta_seq == 0 || db_meta.is_none() { + error!("get_database_history cannot find {:?} db_meta", db_id); + continue; + } + let db_meta = db_meta.unwrap(); + // if include drop db, then no need to fill out of retention time db + if is_drop_time_out_of_retention_time(&db_meta.drop_on, &now) { + continue; + } + if let Some(db_info) = add_dbinfo_map.get(&db_id) { + warn!( + "get db history db:{:?}, db_id:{:?} has no DbIdListKey", + db_info.name_ident, db_info.ident.db_id + ); + db_info_list.push(db_info.clone()); + } + } + } + } + return Ok(db_info_list); } @@ -2524,6 +2619,31 @@ impl + ?Sized> SchemaApi for KV { } } + // add TableIdListKey if not exist + { + // get table id list from _fd_table_id_list/db_id/table_name + let dbid_tbname_idlist = TableIdListKey { + db_id, + table_name: dbid_tbname.table_name.clone(), + }; + let (tb_id_list_seq, _tb_id_list_opt): (_, Option) = + get_pb_value(self, &dbid_tbname_idlist).await?; + if tb_id_list_seq == 0 { + let mut tb_id_list = TableIdList::new(); + tb_id_list.append(table_id); + + warn!( + "drop table:{:?}, table_id:{:?} has no TableIdList", + dbid_tbname, table_id + ); + + condition.push(txn_cond_seq(&dbid_tbname_idlist, Eq, tb_id_list_seq)); + if_then.push(txn_op_put( + &dbid_tbname_idlist, + serialize_struct(&tb_id_list)?, + )); + } + } let txn_req = TxnRequest { condition, if_then, diff --git a/src/meta/api/src/schema_api_test_suite.rs b/src/meta/api/src/schema_api_test_suite.rs index 02e6fd8c6266..5e11cb977b1b 100644 --- a/src/meta/api/src/schema_api_test_suite.rs +++ b/src/meta/api/src/schema_api_test_suite.rs @@ -124,6 +124,7 @@ use databend_common_meta_types::UpsertKV; use log::debug; use log::info; +use crate::deserialize_struct; use crate::is_all_db_data_removed; use crate::kv_app_error::KVAppError; use crate::serialize_struct; @@ -286,6 +287,10 @@ impl SchemaApiTestSuite { suite .table_drop_without_db_id_to_name(&b.build().await) .await?; + suite.list_db_without_db_id_list(&b.build().await).await?; + suite + .drop_table_without_table_id_list(&b.build().await) + .await?; suite.table_rename(&b.build().await).await?; suite.table_update_meta(&b.build().await).await?; suite.table_update_mask_policy(&b.build().await).await?; @@ -1910,6 +1915,118 @@ impl SchemaApiTestSuite { Ok(()) } + #[minitrace::trace] + async fn list_db_without_db_id_list(&self, mt: &MT) -> anyhow::Result<()> + where MT: SchemaApi + kvapi::AsKVApi { + // test drop a db without db_id_list + { + let tenant = "tenant1"; + let db = "db1"; + let mut util = Util::new(mt, tenant, db, "tb2", "JSON"); + util.create_db().await?; + + // remove db id list + let dbid_idlist = DbIdListKey { + tenant: tenant.to_string(), + db_name: db.to_string(), + }; + util.mt + .as_kv_api() + .upsert_kv(UpsertKV::delete(dbid_idlist.to_string_key())) + .await?; + + // drop db + util.drop_db().await?; + + // after drop db, check if db id list has been added + let value = util + .mt + .as_kv_api() + .get_kv(&dbid_idlist.to_string_key()) + .await?; + + assert!(value.is_some()); + let seqv = value.unwrap(); + let db_id_list: DbIdList = deserialize_struct(&seqv.data)?; + assert_eq!(db_id_list.id_list[0], util.db_id); + } + // test get_database_history can return db without db_id_list + { + let tenant = "tenant2"; + let db = "db2"; + let mut util = Util::new(mt, tenant, db, "tb2", "JSON"); + util.create_db().await?; + + // remove db id list + let dbid_idlist = DbIdListKey { + tenant: tenant.to_string(), + db_name: db.to_string(), + }; + util.mt + .as_kv_api() + .upsert_kv(UpsertKV::delete(dbid_idlist.to_string_key())) + .await?; + + let res = mt + .get_database_history(ListDatabaseReq { + tenant: tenant.to_string(), + filter: None, + }) + .await?; + + // check if get_database_history return db_id + let mut found = false; + for db_info in res { + if db_info.ident.db_id == util.db_id { + found = true; + break; + } + } + + assert!(found); + } + Ok(()) + } + + #[minitrace::trace] + async fn drop_table_without_table_id_list(&self, mt: &MT) -> anyhow::Result<()> + where MT: SchemaApi + kvapi::AsKVApi { + // test drop a table without table_id_list + let tenant = "tenant1"; + let db = "db1"; + let table = "tb1"; + let mut util = Util::new(mt, tenant, db, table, "JSON"); + util.create_db().await?; + let (tid, _table_meta) = util.create_table().await?; + + // remove db id list + let table_id_idlist = TableIdListKey { + db_id: util.db_id, + table_name: table.to_string(), + }; + util.mt + .as_kv_api() + .upsert_kv(UpsertKV::delete(table_id_idlist.to_string_key())) + .await?; + + // drop table + util.drop_table_by_id().await?; + + // after drop table, check if table id list has been added + let value = util + .mt + .as_kv_api() + .get_kv(&table_id_idlist.to_string_key()) + .await?; + + assert!(value.is_some()); + let seqv = value.unwrap(); + let id_list: TableIdList = deserialize_struct(&seqv.data)?; + assert_eq!(id_list.id_list[0], tid); + + Ok(()) + } + #[minitrace::trace] async fn table_rename(&self, mt: &MT) -> anyhow::Result<()> { let tenant = "tenant1"; From b32b39c3cdc3e75cc4c8cddc87dd2597c50454ab Mon Sep 17 00:00:00 2001 From: Yijun Zhao Date: Mon, 18 Dec 2023 21:31:58 +0800 Subject: [PATCH 12/20] chore(tests): add explain for sync agg index (#14053) add explain for sync agg index --- .../it/aggregating_index/index_refresh.rs | 5 -- .../it/storages/testdata/settings_table.txt | 4 +- src/query/settings/src/settings_default.rs | 4 +- .../01_ee_system/01_0002_virtual_column.test | 6 --- .../02_0002_sync_agg_index_base.test | 6 --- .../mode/standalone/ee/explain_agg_index.test | 48 +++++++++++++++++++ 6 files changed, 52 insertions(+), 21 deletions(-) diff --git a/src/query/ee/tests/it/aggregating_index/index_refresh.rs b/src/query/ee/tests/it/aggregating_index/index_refresh.rs index bb988368ab95..1ff23ab9b72e 100644 --- a/src/query/ee/tests/it/aggregating_index/index_refresh.rs +++ b/src/query/ee/tests/it/aggregating_index/index_refresh.rs @@ -303,11 +303,6 @@ async fn test_sync_agg_index_after_update() -> Result<()> { .get_settings() .set_enable_refresh_aggregating_index_after_write(true)?; - // ctx.get_settings() - // .set_enable_refresh_aggregating_index_after_write(true)?; - // let fixture = TestFixture::new_with_ctx(_guard, ctx).await; - // let ctx = fixture.ctx(); - // Create table fixture .execute_command("CREATE TABLE t0 (a int, b int, c int) storage_format = 'parquet'") diff --git a/src/query/service/tests/it/storages/testdata/settings_table.txt b/src/query/service/tests/it/storages/testdata/settings_table.txt index 6b5ed2231ddd..1c881a841d6b 100644 --- a/src/query/service/tests/it/storages/testdata/settings_table.txt +++ b/src/query/service/tests/it/storages/testdata/settings_table.txt @@ -31,8 +31,8 @@ DB.Table: 'system'.'settings', Table: settings-table_id:1, ver:0, Engine: System | 'enable_query_profiling' | '0' | '0' | '[0, 1]' | 'SESSION' | 'Enables recording query profile' | 'UInt64' | | 'enable_query_result_cache' | '0' | '0' | '[0, 1]' | 'SESSION' | 'Enables caching query results to improve performance for identical queries.' | 'UInt64' | | 'enable_recluster_after_write' | '1' | '1' | 'None' | 'SESSION' | 'Enables re-clustering after write(copy/replace-into).' | 'UInt64' | -| 'enable_refresh_aggregating_index_after_write' | '0' | '0' | '[0, 1]' | 'SESSION' | 'Refresh aggregating index after new data written' | 'UInt64' | -| 'enable_refresh_virtual_column_after_write' | '0' | '0' | '[0, 1]' | 'SESSION' | 'Refresh virtual column after new data written' | 'UInt64' | +| 'enable_refresh_aggregating_index_after_write' | '1' | '1' | '[0, 1]' | 'SESSION' | 'Refresh aggregating index after new data written' | 'UInt64' | +| 'enable_refresh_virtual_column_after_write' | '1' | '1' | '[0, 1]' | 'SESSION' | 'Refresh virtual column after new data written' | 'UInt64' | | 'enable_replace_into_bloom_pruning' | '1' | '1' | '[0, 1]' | 'SESSION' | 'Enables bloom pruning for replace-into statement.' | 'UInt64' | | 'enable_replace_into_partitioning' | '1' | '1' | '[0, 1]' | 'SESSION' | 'Enables partitioning for replace-into statement (if table has cluster keys).' | 'UInt64' | | 'enable_runtime_filter' | '0' | '0' | '[0, 1]' | 'SESSION' | 'Enables runtime filter optimization for JOIN.' | 'UInt64' | diff --git a/src/query/settings/src/settings_default.rs b/src/query/settings/src/settings_default.rs index 8dd2acccae98..94c36aca0bf8 100644 --- a/src/query/settings/src/settings_default.rs +++ b/src/query/settings/src/settings_default.rs @@ -528,7 +528,7 @@ impl DefaultSettings { range: None, }), ("enable_refresh_aggregating_index_after_write", DefaultSettingValue { - value: UserSettingValue::UInt64(0), + value: UserSettingValue::UInt64(1), desc: "Refresh aggregating index after new data written", mode: SettingMode::Both, range: Some(SettingRange::Numeric(0..=1)), @@ -612,7 +612,7 @@ impl DefaultSettings { range: Some(SettingRange::String(vec!["None", "LZ4", "ZSTD"])), }), ("enable_refresh_virtual_column_after_write", DefaultSettingValue { - value: UserSettingValue::UInt64(0), + value: UserSettingValue::UInt64(1), desc: "Refresh virtual column after new data written", mode: SettingMode::Both, range: Some(SettingRange::Numeric(0..=1)), diff --git a/tests/sqllogictests/suites/ee/01_ee_system/01_0002_virtual_column.test b/tests/sqllogictests/suites/ee/01_ee_system/01_0002_virtual_column.test index 80c8346371fb..c819ec4146ef 100644 --- a/tests/sqllogictests/suites/ee/01_ee_system/01_0002_virtual_column.test +++ b/tests/sqllogictests/suites/ee/01_ee_system/01_0002_virtual_column.test @@ -21,9 +21,6 @@ CREATE DATABASE test_virtual_column statement ok USE test_virtual_column -statement ok -SET enable_refresh_virtual_column_after_write=1; - statement ok drop table if exists t1 @@ -222,9 +219,6 @@ SELECT r.id, r.val['a'], r.nval:a FROM ( SELECT r.id, r.val, r.val as nval FROM 5 55 55 6 NULL NULL -statement ok -SET enable_refresh_virtual_column_after_write=0; - statement ok DROP DATABASE test_virtual_column diff --git a/tests/sqllogictests/suites/ee/02_ee_aggregating_index/02_0002_sync_agg_index_base.test b/tests/sqllogictests/suites/ee/02_ee_aggregating_index/02_0002_sync_agg_index_base.test index 5d1c68dac752..d034ef33c7a2 100644 --- a/tests/sqllogictests/suites/ee/02_ee_aggregating_index/02_0002_sync_agg_index_base.test +++ b/tests/sqllogictests/suites/ee/02_ee_aggregating_index/02_0002_sync_agg_index_base.test @@ -25,9 +25,6 @@ use test_sync_agg_index statement ok DROP AGGREGATING INDEX IF EXISTS testi; -statement ok -SET enable_refresh_aggregating_index_after_write=1; - statement ok CREATE TABLE t (a int, b int, c int) @@ -48,9 +45,6 @@ SELECT b, SUM(a) from t WHERE c > 1 GROUP BY b ORDER BY b 2 3 3 1 -statement ok -SET enable_refresh_aggregating_index_after_write=0; - statement ok DROP AGGREGATING INDEX testi diff --git a/tests/sqllogictests/suites/mode/standalone/ee/explain_agg_index.test b/tests/sqllogictests/suites/mode/standalone/ee/explain_agg_index.test index 284f963524e6..4407a9f1e4ba 100644 --- a/tests/sqllogictests/suites/mode/standalone/ee/explain_agg_index.test +++ b/tests/sqllogictests/suites/mode/standalone/ee/explain_agg_index.test @@ -289,6 +289,54 @@ EvalScalar ├── rewritten query: [selection: [index_col_0 (#0)]] └── estimated rows: 0.00 +# for sync aggregating index + +statement ok +DROP AGGREGATING INDEX idx1 + +statement ok +CREATE SYNC AGGREGATING INDEX idx1 AS SELECT a + 1 from t1 + +statement ok +INSERT INTO t1 VALUES (1,1), (1,2), (2,4), (2,5) + +query T +EXPLAIN SELECT avg(a + 1) from t1 +---- +EvalScalar +├── output columns: [(sum((a + 1)) / if((count((a + 1)) = 0), 1, count((a + 1)))) (#5)] +├── expressions: [sum((a + 1)) (#3) / CAST(if(CAST(count((a + 1)) (#4) = 0 AS Boolean NULL), 1, count((a + 1)) (#4)) AS UInt64 NULL)] +├── estimated rows: 1.00 +└── AggregateFinal + ├── output columns: [sum((a + 1)) (#3), count((a + 1)) (#4)] + ├── group by: [] + ├── aggregate functions: [sum(sum_arg_0), count()] + ├── estimated rows: 1.00 + └── AggregatePartial + ├── output columns: [sum((a + 1)) (#3), count((a + 1)) (#4)] + ├── group by: [] + ├── aggregate functions: [sum(sum_arg_0), count()] + ├── estimated rows: 1.00 + └── EvalScalar + ├── output columns: [sum_arg_0 (#2)] + ├── expressions: [t1.a (#0) + 1] + ├── estimated rows: 4.00 + └── TableScan + ├── table: default.test_index_db.t1 + ├── output columns: [a (#0)] + ├── read rows: 4 + ├── read bytes: 47 + ├── partitions total: 1 + ├── partitions scanned: 1 + ├── pruning stats: [segments: , blocks: ] + ├── push downs: [filters: [], limit: NONE] + ├── aggregating index: [SELECT (a + 1) FROM test_index_db.t1] + ├── rewritten query: [selection: [index_col_0 (#0)]] + └── estimated rows: 4.00 + +statement ok +truncate table t1 + # Should not be rewritten query T EXPLAIN SELECT b, a + 1 as x from t1 order by x From a827842b3a0c46fa0416265326c70c9033c319cc Mon Sep 17 00:00:00 2001 From: sundyli <543950155@qq.com> Date: Mon, 18 Dec 2023 16:01:20 -0800 Subject: [PATCH 13/20] chore(query): refactor decimal scalar functions (#14057) * chore(query): refactor decimal scalar functions * chore(query): refactor decimal scalar functions * chore(query): refactor decimal scalar functions * chore(query): refactor decimal scalar functions * chore(query): refactor decimal scalar functions --- src/query/expression/src/types/decimal.rs | 21 +- src/query/expression/src/utils/serialize.rs | 12 +- src/query/expression/src/values.rs | 10 + .../src/aggregates/aggregate_array_moving.rs | 2 +- .../functions/src/aggregates/aggregate_avg.rs | 2 +- .../src/aggregates/aggregate_stddev.rs | 8 +- src/query/functions/src/scalars/arithmetic.rs | 8 +- src/query/functions/src/scalars/decimal.rs | 1911 ----------------- .../src/scalars/decimal/arithmetic.rs | 363 ++++ .../functions/src/scalars/decimal/cast.rs | 746 +++++++ .../src/scalars/decimal/comparison.rs | 130 ++ .../functions/src/scalars/decimal/math.rs | 289 +++ .../functions/src/scalars/decimal/mod.rs | 25 + src/query/functions/src/scalars/mod.rs | 2 +- src/query/functions/tests/it/scalars/cast.rs | 5 + .../functions/tests/it/scalars/comparison.rs | 1 + .../tests/it/scalars/testdata/cast.txt | 18 + .../tests/it/scalars/testdata/comparison.txt | 9 + .../src/parquet_rs/statistics/column.rs | 14 +- .../parquet/src/parquet_rs/statistics/page.rs | 8 +- 20 files changed, 1627 insertions(+), 1957 deletions(-) delete mode 100644 src/query/functions/src/scalars/decimal.rs create mode 100644 src/query/functions/src/scalars/decimal/arithmetic.rs create mode 100644 src/query/functions/src/scalars/decimal/cast.rs create mode 100644 src/query/functions/src/scalars/decimal/comparison.rs create mode 100644 src/query/functions/src/scalars/decimal/math.rs create mode 100644 src/query/functions/src/scalars/decimal/mod.rs diff --git a/src/query/expression/src/types/decimal.rs b/src/query/expression/src/types/decimal.rs index d27cc86488d0..91233032fe16 100644 --- a/src/query/expression/src/types/decimal.rs +++ b/src/query/expression/src/types/decimal.rs @@ -26,7 +26,6 @@ use ethnum::i256; use ethnum::AsI256; use itertools::Itertools; use num_traits::NumCast; -use num_traits::ToPrimitive; use serde::Deserialize; use serde::Serialize; @@ -305,8 +304,8 @@ pub trait Decimal: fn default_decimal_size() -> DecimalSize; fn from_float(value: f64) -> Self; - fn from_u64(value: u64) -> Self; - fn from_i64(value: i64) -> Self; + fn from_i128>(value: U) -> Self; + fn de_binary(bytes: &mut &[u8]) -> Self; fn to_float32(self, scale: u8) -> f32; @@ -442,12 +441,8 @@ impl Decimal for i128 { } } - fn from_u64(value: u64) -> Self { - value.to_i128().unwrap() - } - - fn from_i64(value: i64) -> Self { - value.to_i128().unwrap() + fn from_i128>(value: U) -> Self { + value.into() } fn de_binary(bytes: &mut &[u8]) -> Self { @@ -611,12 +606,8 @@ impl Decimal for i256 { value.as_i256() } - fn from_u64(value: u64) -> Self { - i256::from(value.to_i128().unwrap()) - } - - fn from_i64(value: i64) -> Self { - i256::from(value.to_i128().unwrap()) + fn from_i128>(value: U) -> Self { + i256::from(value.into()) } fn de_binary(bytes: &mut &[u8]) -> Self { diff --git a/src/query/expression/src/utils/serialize.rs b/src/query/expression/src/utils/serialize.rs index b95a70d53662..3b8831f64a15 100644 --- a/src/query/expression/src/utils/serialize.rs +++ b/src/query/expression/src/utils/serialize.rs @@ -50,9 +50,9 @@ pub fn read_decimal_with_size( // Checking whether numbers need to be added or subtracted to calculate rounding if let Some(r) = n.checked_rem(T::e(scale_diff)) { if let Some(m) = r.checked_div(T::e(scale_diff - 1)) { - if m >= T::from_i64(5i64) { + if m >= T::from_i128(5i64) { round_val = Some(T::one()); - } else if m <= T::from_i64(-5i64) { + } else if m <= T::from_i128(-5i64) { round_val = Some(T::minus_one()); } } @@ -140,7 +140,7 @@ pub fn read_decimal( .checked_mul(T::e(zeros + 1)) .ok_or_else(decimal_overflow_error)?; n = n - .checked_add(T::from_u64((v - b'0') as u64)) + .checked_add(T::from_i128((v - b'0') as u64)) .ok_or_else(decimal_overflow_error)?; zeros = 0; } @@ -200,7 +200,7 @@ pub fn read_decimal( .checked_mul(T::e(zeros + 1)) .ok_or_else(decimal_overflow_error)?; n = n - .checked_add(T::from_u64((v - b'0') as u64)) + .checked_add(T::from_i128((v - b'0') as u64)) .ok_or_else(decimal_overflow_error)?; digits += zeros + 1; zeros = 0; @@ -288,11 +288,11 @@ pub fn read_decimal_from_json( match value { serde_json::Value::Number(n) => { if n.is_i64() { - Ok(T::from_i64(n.as_i64().unwrap()) + Ok(T::from_i128(n.as_i64().unwrap()) .with_size(size) .ok_or_else(decimal_overflow_error)?) } else if n.is_u64() { - Ok(T::from_u64(n.as_u64().unwrap()) + Ok(T::from_i128(n.as_u64().unwrap()) .with_size(size) .ok_or_else(decimal_overflow_error)?) } else { diff --git a/src/query/expression/src/values.rs b/src/query/expression/src/values.rs index 200460ad075f..2a43d27c6767 100755 --- a/src/query/expression/src/values.rs +++ b/src/query/expression/src/values.rs @@ -59,6 +59,7 @@ use crate::types::decimal::DecimalColumnBuilder; use crate::types::decimal::DecimalDataType; use crate::types::decimal::DecimalScalar; use crate::types::decimal::DecimalSize; +use crate::types::decimal::DecimalType; use crate::types::nullable::NullableColumn; use crate::types::nullable::NullableColumnBuilder; use crate::types::nullable::NullableColumnVec; @@ -284,6 +285,15 @@ impl Value { } } +impl Value> { + pub fn upcast_decimal(self, size: DecimalSize) -> Value { + match self { + Value::Scalar(scalar) => Value::Scalar(T::upcast_scalar(scalar, size)), + Value::Column(col) => Value::Column(T::upcast_column(col, size)), + } + } +} + impl Value { pub fn convert_to_full_column(&self, ty: &DataType, num_rows: usize) -> Column { match self { diff --git a/src/query/functions/src/aggregates/aggregate_array_moving.rs b/src/query/functions/src/aggregates/aggregate_array_moving.rs index 23efb0eea025..acffce28f419 100644 --- a/src/query/functions/src/aggregates/aggregate_array_moving.rs +++ b/src/query/functions/src/aggregates/aggregate_array_moving.rs @@ -374,7 +374,7 @@ where T: Decimal } let avg_val = match sum .checked_mul(T::e(scale_add as u32)) - .and_then(|v| v.checked_div(T::from_u64(window_size as u64))) + .and_then(|v| v.checked_div(T::from_i128(window_size as u64))) { Some(value) => value, None => { diff --git a/src/query/functions/src/aggregates/aggregate_avg.rs b/src/query/functions/src/aggregates/aggregate_avg.rs index 5304f9751ed4..94cd8dfff9c9 100644 --- a/src/query/functions/src/aggregates/aggregate_avg.rs +++ b/src/query/functions/src/aggregates/aggregate_avg.rs @@ -189,7 +189,7 @@ where match self .value .checked_mul(T::Scalar::e(decimal_avg_data.scale_add as u32)) - .and_then(|v| v.checked_div(T::Scalar::from_u64(self.count))) + .and_then(|v| v.checked_div(T::Scalar::from_i128(self.count))) { Some(value) => { T::push_item(builder, T::to_scalar_ref(&value)); diff --git a/src/query/functions/src/aggregates/aggregate_stddev.rs b/src/query/functions/src/aggregates/aggregate_stddev.rs index 2800e2423cf0..573567aaa91a 100644 --- a/src/query/functions/src/aggregates/aggregate_stddev.rs +++ b/src/query/functions/src/aggregates/aggregate_stddev.rs @@ -173,7 +173,7 @@ where self.count += 1; if self.count > 1 { let t = match value - .checked_mul(T::Scalar::from_u64(self.count)) + .checked_mul(T::Scalar::from_i128(self.count)) .and_then(|v| v.checked_sub(self.sum)) .and_then(|v| v.checked_mul(T::Scalar::e(VARIANCE_PRECISION as u32))) { @@ -204,7 +204,7 @@ where } }; - let count = T::Scalar::from_u64(self.count * (self.count - 1)); + let count = T::Scalar::from_i128(self.count * (self.count - 1)); let add_variance = match t.checked_div(count) { Some(t) => t, @@ -236,8 +236,8 @@ where return Ok(()); } - let other_count = T::Scalar::from_u64(other.count); - let self_count = T::Scalar::from_u64(self.count); + let other_count = T::Scalar::from_i128(other.count); + let self_count = T::Scalar::from_i128(self.count); let t = match other_count .checked_mul(self.sum) .and_then(|v| v.checked_mul(T::Scalar::e(VARIANCE_PRECISION as u32))) diff --git a/src/query/functions/src/scalars/arithmetic.rs b/src/query/functions/src/scalars/arithmetic.rs index 4d75ecb3e189..1319dacc70e6 100644 --- a/src/query/functions/src/scalars/arithmetic.rs +++ b/src/query/functions/src/scalars/arithmetic.rs @@ -43,6 +43,7 @@ use databend_common_expression::types::ALL_INTEGER_TYPES; use databend_common_expression::types::ALL_NUMBER_CLASSES; use databend_common_expression::types::ALL_NUMERICS_TYPES; use databend_common_expression::types::ALL_UNSIGNED_INTEGER_TYPES; +use databend_common_expression::types::F32; use databend_common_expression::utils::arithmetics_type::ResultTypeOfBinary; use databend_common_expression::utils::arithmetics_type::ResultTypeOfUnary; use databend_common_expression::values::Value; @@ -72,10 +73,9 @@ use lexical_core::FormattedSize; use num_traits::AsPrimitive; use super::arithmetic_modulo::vectorize_modulo; -use super::decimal::register_decimal_to_float32; -use super::decimal::register_decimal_to_float64; use super::decimal::register_decimal_to_int; use crate::scalars::decimal::register_decimal_arithmetic; +use crate::scalars::decimal::register_decimal_to_float; pub fn register(registry: &mut FunctionRegistry) { registry.register_aliases("plus", &["add"]); @@ -717,10 +717,10 @@ pub fn register_number_to_number(registry: &mut FunctionRegistry) { NumberClass::Decimal128 => { // todo(youngsofun): add decimal try_cast and decimal to int and float if matches!(dest_type, NumberDataType::Float32) { - register_decimal_to_float32(registry); + register_decimal_to_float::(registry); } if matches!(dest_type, NumberDataType::Float64) { - register_decimal_to_float64(registry); + register_decimal_to_float::(registry); } with_number_mapped_type!(|DEST_TYPE| match dest_type { diff --git a/src/query/functions/src/scalars/decimal.rs b/src/query/functions/src/scalars/decimal.rs deleted file mode 100644 index 78618cc8720e..000000000000 --- a/src/query/functions/src/scalars/decimal.rs +++ /dev/null @@ -1,1911 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::cmp::Ord; -use std::ops::*; -use std::sync::Arc; - -use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_arrow::arrow::buffer::Buffer; -use databend_common_expression::serialize::read_decimal_with_size; -use databend_common_expression::type_check::common_super_type; -use databend_common_expression::types::decimal::*; -use databend_common_expression::types::string::StringColumn; -use databend_common_expression::types::*; -use databend_common_expression::with_decimal_mapped_type; -use databend_common_expression::with_integer_mapped_type; -use databend_common_expression::with_number_mapped_type; -use databend_common_expression::Column; -use databend_common_expression::ColumnBuilder; -use databend_common_expression::Domain; -use databend_common_expression::EvalContext; -use databend_common_expression::FromData; -use databend_common_expression::Function; -use databend_common_expression::FunctionContext; -use databend_common_expression::FunctionDomain; -use databend_common_expression::FunctionEval; -use databend_common_expression::FunctionRegistry; -use databend_common_expression::FunctionSignature; -use databend_common_expression::Scalar; -use databend_common_expression::ScalarRef; -use databend_common_expression::SimpleDomainCmp; -use databend_common_expression::Value; -use databend_common_expression::ValueRef; -use ethnum::i256; -use num_traits::AsPrimitive; -use ordered_float::OrderedFloat; - -macro_rules! op_decimal { - ($a: expr, $b: expr, $ctx: expr, $left: expr, $right: expr, $result_type: expr, $op: ident, $is_divide: expr) => { - match $left { - DecimalDataType::Decimal128(_) => { - binary_decimal!( - $a, - $b, - $ctx, - $left, - $right, - $op, - $result_type.size(), - i128, - Decimal128, - $is_divide - ) - } - DecimalDataType::Decimal256(_) => { - binary_decimal!( - $a, - $b, - $ctx, - $left, - $right, - $op, - $result_type.size(), - i256, - Decimal256, - $is_divide - ) - } - } - }; - ($a: expr, $b: expr, $return_type: expr, $op: ident) => { - match $return_type { - DecimalDataType::Decimal128(_) => { - compare_decimal!($a, $b, $op, Decimal128) - } - DecimalDataType::Decimal256(_) => { - compare_decimal!($a, $b, $op, Decimal256) - } - } - }; -} - -macro_rules! compare_decimal { - ($a: expr, $b: expr, $op: ident, $decimal_type: tt) => {{ - match ($a, $b) { - ( - ValueRef::Column(Column::Decimal(DecimalColumn::$decimal_type(buffer_a, _))), - ValueRef::Column(Column::Decimal(DecimalColumn::$decimal_type(buffer_b, _))), - ) => { - let result = buffer_a - .iter() - .zip(buffer_b.iter()) - .map(|(a, b)| a.cmp(b).$op()) - .collect(); - - Value::Column(Column::Boolean(result)) - } - - ( - ValueRef::Column(Column::Decimal(DecimalColumn::$decimal_type(buffer, _))), - ValueRef::Scalar(ScalarRef::Decimal(DecimalScalar::$decimal_type(b, _))), - ) => { - let result = buffer.iter().map(|a| a.cmp(b).$op()).collect(); - - Value::Column(Column::Boolean(result)) - } - - ( - ValueRef::Scalar(ScalarRef::Decimal(DecimalScalar::$decimal_type(a, _))), - ValueRef::Column(Column::Decimal(DecimalColumn::$decimal_type(buffer, _))), - ) => { - let result = buffer.iter().map(|b| a.cmp(b).$op()).collect(); - - Value::Column(Column::Boolean(result)) - } - - ( - ValueRef::Scalar(ScalarRef::Decimal(DecimalScalar::$decimal_type(a, _))), - ValueRef::Scalar(ScalarRef::Decimal(DecimalScalar::$decimal_type(b, _))), - ) => Value::Scalar(Scalar::Boolean(a.cmp(b).$op())), - - _ => unreachable!("arg type of cmp op is not required decimal"), - } - }}; -} - -macro_rules! binary_decimal { - ($a: expr, $b: expr, $ctx: expr, $left: expr, $right: expr, $op: ident, $size: expr, $type_name: ty, $decimal_type: tt, $is_divide: expr) => {{ - let overflow = $size.precision == <$type_name>::default_decimal_size().precision; - - if $is_divide { - let scale_a = $left.scale(); - let scale_b = $right.scale(); - binary_decimal_div!( - $a, - $b, - $ctx, - scale_a, - scale_b, - $op, - $size, - $type_name, - $decimal_type - ) - } else if overflow { - binary_decimal_check_overflow!($a, $b, $ctx, $op, $size, $type_name, $decimal_type) - } else { - binary_decimal_no_overflow!($a, $b, $ctx, $op, $size, $type_name, $decimal_type) - } - }}; -} - -macro_rules! binary_decimal_no_overflow { - ($a: expr, $b: expr, $ctx: expr, $op: ident, $size: expr, $type_name: ty, $decimal_type: tt) => {{ - match ($a, $b) { - ( - ValueRef::Column(Column::Decimal(DecimalColumn::$decimal_type(buffer_a, _))), - ValueRef::Column(Column::Decimal(DecimalColumn::$decimal_type(buffer_b, _))), - ) => { - let result: Vec<_> = buffer_a - .iter() - .zip(buffer_b.iter()) - .map(|(a, b)| a.$op(b)) - .collect(); - Value::Column(Column::Decimal(DecimalColumn::$decimal_type( - result.into(), - $size, - ))) - } - - ( - ValueRef::Column(Column::Decimal(DecimalColumn::$decimal_type(buffer, _))), - ValueRef::Scalar(ScalarRef::Decimal(DecimalScalar::$decimal_type(b, _))), - ) => { - let result: Vec<_> = buffer.iter().map(|a| a.$op(b)).collect(); - - Value::Column(Column::Decimal(DecimalColumn::$decimal_type( - result.into(), - $size, - ))) - } - - ( - ValueRef::Scalar(ScalarRef::Decimal(DecimalScalar::$decimal_type(a, _))), - ValueRef::Column(Column::Decimal(DecimalColumn::$decimal_type(buffer, _))), - ) => { - let result: Vec<_> = buffer.iter().map(|b| a.$op(b)).collect(); - Value::Column(Column::Decimal(DecimalColumn::$decimal_type( - result.into(), - $size, - ))) - } - - ( - ValueRef::Scalar(ScalarRef::Decimal(DecimalScalar::$decimal_type(a, _))), - ValueRef::Scalar(ScalarRef::Decimal(DecimalScalar::$decimal_type(b, _))), - ) => Value::Scalar(Scalar::Decimal(DecimalScalar::$decimal_type( - a.$op(b), - $size, - ))), - - _ => unreachable!("arg type of binary op is not required decimal"), - } - }}; -} - -macro_rules! binary_decimal_check_overflow { - ($a: expr, $b: expr, $ctx: expr, $op: ident, $size: expr, $type_name: ty, $decimal_type: tt) => {{ - let one = <$type_name>::one(); - let min_for_precision = <$type_name>::min_for_precision($size.precision); - let max_for_precision = <$type_name>::max_for_precision($size.precision); - - match ($a, $b) { - ( - ValueRef::Column(Column::Decimal(DecimalColumn::$decimal_type(buffer_a, _))), - ValueRef::Column(Column::Decimal(DecimalColumn::$decimal_type(buffer_b, _))), - ) => { - let mut result = Vec::with_capacity(buffer_a.len()); - - for (a, b) in buffer_a.iter().zip(buffer_b.iter()) { - let t = a.$op(b); - if t < min_for_precision || t > max_for_precision { - $ctx.set_error( - result.len(), - concat!("Decimal overflow at line : ", line!()), - ); - result.push(one); - } else { - result.push(t); - } - } - Value::Column(Column::Decimal(DecimalColumn::$decimal_type( - result.into(), - $size, - ))) - } - - ( - ValueRef::Column(Column::Decimal(DecimalColumn::$decimal_type(buffer, _))), - ValueRef::Scalar(ScalarRef::Decimal(DecimalScalar::$decimal_type(b, _))), - ) => { - let mut result = Vec::with_capacity(buffer.len()); - - for a in buffer.iter() { - let t = a.$op(b); - if t < min_for_precision || t > max_for_precision { - $ctx.set_error( - result.len(), - concat!("Decimal overflow at line : ", line!()), - ); - result.push(one); - } else { - result.push(t); - } - } - - Value::Column(Column::Decimal(DecimalColumn::$decimal_type( - result.into(), - $size, - ))) - } - - ( - ValueRef::Scalar(ScalarRef::Decimal(DecimalScalar::$decimal_type(a, _))), - ValueRef::Column(Column::Decimal(DecimalColumn::$decimal_type(buffer, _))), - ) => { - let mut result = Vec::with_capacity(buffer.len()); - - for b in buffer.iter() { - let t = a.$op(b); - if t < min_for_precision || t > max_for_precision { - $ctx.set_error( - result.len(), - concat!("Decimal overflow at line : ", line!()), - ); - result.push(one); - } else { - result.push(t); - } - } - Value::Column(Column::Decimal(DecimalColumn::$decimal_type( - result.into(), - $size, - ))) - } - - ( - ValueRef::Scalar(ScalarRef::Decimal(DecimalScalar::$decimal_type(a, _))), - ValueRef::Scalar(ScalarRef::Decimal(DecimalScalar::$decimal_type(b, _))), - ) => { - let t = a.$op(b); - if t < min_for_precision || t > max_for_precision { - $ctx.set_error(0, concat!("Decimal overflow at line : ", line!())); - } - Value::Scalar(Scalar::Decimal(DecimalScalar::$decimal_type(t, $size))) - } - - _ => unreachable!("arg type of binary op is not required decimal"), - } - }}; -} - -macro_rules! binary_decimal_div { - ($a: expr, $b: expr, $ctx: expr, $scale_a: expr, $scale_b: expr, $op: ident, $size: expr, $type_name: ty, $decimal_type: tt) => {{ - let zero = <$type_name>::zero(); - let one = <$type_name>::one(); - - let (scale_mul, scale_div) = if $scale_b + $size.scale > $scale_a { - ($scale_b + $size.scale - $scale_a, 0) - } else { - (0, $scale_b + $size.scale - $scale_a) - }; - - let multiplier = <$type_name>::e(scale_mul as u32); - let div = <$type_name>::e(scale_div as u32); - - match ($a, $b) { - ( - ValueRef::Column(Column::Decimal(DecimalColumn::$decimal_type(buffer_a, _))), - ValueRef::Column(Column::Decimal(DecimalColumn::$decimal_type(buffer_b, _))), - ) => { - let mut result = Vec::with_capacity(buffer_a.len()); - - for (a, b) in buffer_a.iter().zip(buffer_b.iter()) { - if std::intrinsics::unlikely(*b == zero) { - $ctx.set_error(result.len(), "divided by zero"); - result.push(one); - } else { - result.push((a * multiplier).$op(b) / div); - } - } - Value::Column(Column::Decimal(DecimalColumn::$decimal_type( - result.into(), - $size, - ))) - } - - ( - ValueRef::Column(Column::Decimal(DecimalColumn::$decimal_type(buffer, _))), - ValueRef::Scalar(ScalarRef::Decimal(DecimalScalar::$decimal_type(b, _))), - ) => { - let mut result = Vec::with_capacity(buffer.len()); - - for a in buffer.iter() { - if std::intrinsics::unlikely(*b == zero) { - $ctx.set_error(result.len(), "divided by zero"); - result.push(one); - } else { - result.push((a * multiplier).$op(b) / div); - } - } - - Value::Column(Column::Decimal(DecimalColumn::$decimal_type( - result.into(), - $size, - ))) - } - - ( - ValueRef::Scalar(ScalarRef::Decimal(DecimalScalar::$decimal_type(a, _))), - ValueRef::Column(Column::Decimal(DecimalColumn::$decimal_type(buffer, _))), - ) => { - let mut result = Vec::with_capacity(buffer.len()); - - for b in buffer.iter() { - if std::intrinsics::unlikely(*b == zero) { - $ctx.set_error(result.len(), "divided by zero"); - result.push(one); - } else { - result.push((a * multiplier).$op(b) / div); - } - } - Value::Column(Column::Decimal(DecimalColumn::$decimal_type( - result.into(), - $size, - ))) - } - - ( - ValueRef::Scalar(ScalarRef::Decimal(DecimalScalar::$decimal_type(a, _))), - ValueRef::Scalar(ScalarRef::Decimal(DecimalScalar::$decimal_type(b, _))), - ) => { - let mut t = zero; - if std::intrinsics::unlikely(*b == zero) { - $ctx.set_error(0, "divided by zero"); - } else { - t = (a * multiplier).$op(b) / div; - } - Value::Scalar(Scalar::Decimal(DecimalScalar::$decimal_type(t, $size))) - } - - _ => unreachable!("arg type of binary op is not required decimal"), - } - }}; -} - -macro_rules! register_decimal_compare_op { - ($registry: expr, $name: expr, $op: ident, $domain_op: tt) => { - $registry.register_function_factory($name, |_, args_type| { - if args_type.len() != 2 { - return None; - } - - let has_nullable = args_type.iter().any(|x| x.is_nullable_or_null()); - let args_type: Vec = args_type.iter().map(|x| x.remove_nullable()).collect(); - - // Only works for one of is decimal types - if !args_type[0].is_decimal() && !args_type[1].is_decimal() { - return None; - } - - let common_type = common_super_type(args_type[0].clone(), args_type[1].clone(), &[])?; - - if !common_type.is_decimal() { - return None; - } - - // Comparison between different decimal types must be same siganature types - let function = Function { - signature: FunctionSignature { - name: $name.to_string(), - args_type: vec![common_type.clone(), common_type.clone()], - return_type: DataType::Boolean, - }, - eval: FunctionEval::Scalar { - calc_domain: Box::new(|_, d| { - let new_domain = match (&d[0], &d[1]) { - ( - Domain::Decimal(DecimalDomain::Decimal128(d1, _)), - Domain::Decimal(DecimalDomain::Decimal128(d2, _)), - ) => d1.$domain_op(d2), - ( - Domain::Decimal(DecimalDomain::Decimal256(d1, _)), - Domain::Decimal(DecimalDomain::Decimal256(d2, _)), - ) => d1.$domain_op(d2), - _ => unreachable!("Expect two same decimal domains, got {:?}", d), - }; - new_domain.map(|d| Domain::Boolean(d)) - }), - eval: Box::new(move |args, _ctx| { - op_decimal!(&args[0], &args[1], common_type.as_decimal().unwrap(), $op) - }), - }, - }; - if has_nullable { - Some(Arc::new(function.passthrough_nullable())) - } else { - Some(Arc::new(function)) - } - }); - }; -} - -#[inline(always)] -fn domain_plus( - lhs: &SimpleDomain, - rhs: &SimpleDomain, - precision: u8, -) -> Option> { - // For plus, the scale of the two operands must be the same. - let min = T::min_for_precision(precision); - let max = T::max_for_precision(precision); - Some(SimpleDomain { - min: lhs - .min - .checked_add(rhs.min) - .filter(|&m| m >= min && m <= max)?, - max: lhs - .max - .checked_add(rhs.max) - .filter(|&m| m >= min && m <= max)?, - }) -} - -#[inline(always)] -fn domain_minus( - lhs: &SimpleDomain, - rhs: &SimpleDomain, - precision: u8, -) -> Option> { - // For minus, the scale of the two operands must be the same. - let min = T::min_for_precision(precision); - let max = T::max_for_precision(precision); - Some(SimpleDomain { - min: lhs - .min - .checked_sub(rhs.max) - .filter(|&m| m >= min && m <= max)?, - max: lhs - .max - .checked_sub(rhs.min) - .filter(|&m| m >= min && m <= max)?, - }) -} - -#[inline(always)] -fn domain_mul( - lhs: &SimpleDomain, - rhs: &SimpleDomain, - precision: u8, -) -> Option> { - let min = T::min_for_precision(precision); - let max = T::max_for_precision(precision); - - let a = lhs - .min - .checked_mul(rhs.min) - .filter(|&m| m >= min && m <= max)?; - let b = lhs - .min - .checked_mul(rhs.max) - .filter(|&m| m >= min && m <= max)?; - let c = lhs - .max - .checked_mul(rhs.min) - .filter(|&m| m >= min && m <= max)?; - let d = lhs - .max - .checked_mul(rhs.max) - .filter(|&m| m >= min && m <= max)?; - - Some(SimpleDomain { - min: a.min(b).min(c).min(d), - max: a.max(b).max(c).max(d), - }) -} - -#[inline(always)] -fn domain_div( - _lhs: &SimpleDomain, - _rhs: &SimpleDomain, - _precision: u8, -) -> Option> { - // For div, we cannot determine the domain. - None -} - -macro_rules! register_decimal_binary_op { - ($registry: expr, $name: expr, $op: ident, $domain_op: ident, $default_domain: expr) => { - $registry.register_function_factory($name, |_, args_type| { - if args_type.len() != 2 { - return None; - } - - let has_nullable = args_type.iter().any(|x| x.is_nullable_or_null()); - let args_type: Vec = args_type.iter().map(|x| x.remove_nullable()).collect(); - - // number X decimal -> decimal - // decimal X number -> decimal - // decimal X decimal -> decimal - if !args_type[0].is_decimal() && !args_type[1].is_decimal() { - return None; - } - - let decimal_a = - DecimalDataType::from_size(args_type[0].get_decimal_properties()?).unwrap(); - let decimal_b = - DecimalDataType::from_size(args_type[1].get_decimal_properties()?).unwrap(); - - let is_multiply = $name == "multiply"; - let is_divide = $name == "divide"; - let is_plus_minus = !is_multiply && !is_divide; - - // left, right will unify to same width decimal, both 256 or both 128 - let (left, right, return_decimal_type) = DecimalDataType::binary_result_type( - &decimal_a, - &decimal_b, - is_multiply, - is_divide, - is_plus_minus, - ) - .ok()?; - - let function = Function { - signature: FunctionSignature { - name: $name.to_string(), - args_type: vec![ - DataType::Decimal(left.clone()), - DataType::Decimal(right.clone()), - ], - return_type: DataType::Decimal(return_decimal_type), - }, - eval: FunctionEval::Scalar { - calc_domain: Box::new(move |_ctx, d| { - let lhs = d[0].as_decimal(); - let rhs = d[1].as_decimal(); - - if lhs.is_none() || rhs.is_none() { - return FunctionDomain::Full; - } - - let lhs = lhs.unwrap(); - let rhs = rhs.unwrap(); - - let size = return_decimal_type.size(); - - { - match (lhs, rhs) { - ( - DecimalDomain::Decimal128(d1, _), - DecimalDomain::Decimal128(d2, _), - ) => $domain_op(&d1, &d2, size.precision) - .map(|d| DecimalDomain::Decimal128(d, size)), - ( - DecimalDomain::Decimal256(d1, _), - DecimalDomain::Decimal256(d2, _), - ) => $domain_op(&d1, &d2, size.precision) - .map(|d| DecimalDomain::Decimal256(d, size)), - _ => { - unreachable!("unreachable decimal domain {:?} /{:?}", lhs, rhs) - } - } - } - .map(|d| FunctionDomain::Domain(Domain::Decimal(d))) - .unwrap_or($default_domain) - }), - eval: Box::new(move |args, ctx| { - let res = op_decimal!( - &args[0], - &args[1], - ctx, - left, - right, - return_decimal_type, - $op, - is_divide - ); - - res - }), - }, - }; - if has_nullable { - Some(Arc::new(function.passthrough_nullable())) - } else { - Some(Arc::new(function)) - } - }); - }; -} - -pub(crate) fn register_decimal_compare_op(registry: &mut FunctionRegistry) { - register_decimal_compare_op!(registry, "lt", is_lt, domain_lt); - register_decimal_compare_op!(registry, "eq", is_eq, domain_eq); - register_decimal_compare_op!(registry, "gt", is_gt, domain_gt); - register_decimal_compare_op!(registry, "lte", is_le, domain_lte); - register_decimal_compare_op!(registry, "gte", is_ge, domain_gte); - register_decimal_compare_op!(registry, "noteq", is_ne, domain_noteq); -} - -pub(crate) fn register_decimal_arithmetic(registry: &mut FunctionRegistry) { - // TODO checked overflow by default - register_decimal_binary_op!(registry, "plus", add, domain_plus, FunctionDomain::Full); - register_decimal_binary_op!(registry, "minus", sub, domain_minus, FunctionDomain::Full); - register_decimal_binary_op!( - registry, - "divide", - div, - domain_div, - FunctionDomain::MayThrow - ); - register_decimal_binary_op!(registry, "multiply", mul, domain_mul, FunctionDomain::Full); -} - -// int float to decimal -pub fn register(registry: &mut FunctionRegistry) { - let factory = |params: &[usize], args_type: &[DataType]| { - if args_type.len() != 1 { - return None; - } - if params.len() != 2 { - return None; - } - - let from_type = args_type[0].remove_nullable(); - - if !matches!( - from_type, - DataType::Boolean | DataType::Number(_) | DataType::Decimal(_) | DataType::String - ) { - return None; - } - - let decimal_size = DecimalSize { - precision: params[0] as u8, - scale: params[1] as u8, - }; - - let decimal_type = DecimalDataType::from_size(decimal_size).ok()?; - - Some(Function { - signature: FunctionSignature { - name: "to_decimal".to_string(), - args_type: vec![from_type.clone()], - return_type: DataType::Decimal(decimal_type), - }, - eval: FunctionEval::Scalar { - calc_domain: Box::new(move |ctx, d| { - convert_to_decimal_domain(ctx, d[0].clone(), decimal_type) - .map(|d| FunctionDomain::Domain(Domain::Decimal(d))) - .unwrap_or(FunctionDomain::MayThrow) - }), - eval: Box::new(move |args, ctx| { - convert_to_decimal(&args[0], ctx, &from_type, decimal_type) - }), - }, - }) - }; - - registry.register_function_factory("to_decimal", move |params, args_type| { - Some(Arc::new(factory(params, args_type)?)) - }); - registry.register_function_factory("to_decimal", move |params, args_type| { - let f = factory(params, args_type)?; - Some(Arc::new(f.passthrough_nullable())) - }); - registry.register_function_factory("try_to_decimal", move |params, args_type| { - let mut f = factory(params, args_type)?; - f.signature.name = "try_to_decimal".to_string(); - Some(Arc::new(f.error_to_null())) - }); - registry.register_function_factory("try_to_decimal", move |params, args_type| { - let mut f = factory(params, args_type)?; - f.signature.name = "try_to_decimal".to_string(); - Some(Arc::new(f.error_to_null().passthrough_nullable())) - }); -} - -pub(crate) fn register_decimal_to_float64(registry: &mut FunctionRegistry) { - let factory = |_params: &[usize], args_type: &[DataType]| { - if args_type.len() != 1 { - return None; - } - - let arg_type = args_type[0].remove_nullable(); - - if !arg_type.is_decimal() { - return None; - } - - let function = Function { - signature: FunctionSignature { - name: "to_float64".to_string(), - args_type: vec![arg_type.clone()], - return_type: Float64Type::data_type(), - }, - eval: FunctionEval::Scalar { - calc_domain: Box::new(|_, d| match d[0].as_decimal().unwrap() { - DecimalDomain::Decimal128(d, size) => FunctionDomain::Domain(Domain::Number( - NumberDomain::Float64(SimpleDomain { - min: OrderedFloat(d.min.to_float64(size.scale)), - max: OrderedFloat(d.max.to_float64(size.scale)), - }), - )), - DecimalDomain::Decimal256(d, size) => FunctionDomain::Domain(Domain::Number( - NumberDomain::Float64(SimpleDomain { - min: OrderedFloat(d.min.to_float64(size.scale)), - max: OrderedFloat(d.max.to_float64(size.scale)), - }), - )), - }), - eval: Box::new(move |args, tx| decimal_to_float64(&args[0], arg_type.clone(), tx)), - }, - }; - - Some(function) - }; - - registry.register_function_factory("to_float64", move |params, args_type| { - Some(Arc::new(factory(params, args_type)?)) - }); - registry.register_function_factory("to_float64", move |params, args_type| { - let f = factory(params, args_type)?; - Some(Arc::new(f.passthrough_nullable())) - }); - registry.register_function_factory("try_to_float64", move |params, args_type| { - let mut f = factory(params, args_type)?; - f.signature.name = "try_to_float64".to_string(); - Some(Arc::new(f.error_to_null())) - }); - registry.register_function_factory("try_to_float64", move |params, args_type| { - let mut f = factory(params, args_type)?; - f.signature.name = "try_to_float64".to_string(); - Some(Arc::new(f.error_to_null().passthrough_nullable())) - }); -} - -pub(crate) fn register_decimal_to_float32(registry: &mut FunctionRegistry) { - let factory = |_params: &[usize], args_type: &[DataType]| { - if args_type.len() != 1 { - return None; - } - - let arg_type = args_type[0].remove_nullable(); - if !arg_type.is_decimal() { - return None; - } - - let function = Function { - signature: FunctionSignature { - name: "to_float32".to_string(), - args_type: vec![arg_type.clone()], - return_type: Float32Type::data_type(), - }, - eval: FunctionEval::Scalar { - calc_domain: Box::new(|_, d| match d[0].as_decimal().unwrap() { - DecimalDomain::Decimal128(d, size) => FunctionDomain::Domain(Domain::Number( - NumberDomain::Float32(SimpleDomain { - min: OrderedFloat(d.min.to_float32(size.scale)), - max: OrderedFloat(d.max.to_float32(size.scale)), - }), - )), - DecimalDomain::Decimal256(d, size) => FunctionDomain::Domain(Domain::Number( - NumberDomain::Float32(SimpleDomain { - min: OrderedFloat(d.min.to_float32(size.scale)), - max: OrderedFloat(d.max.to_float32(size.scale)), - }), - )), - }), - eval: Box::new(move |args, tx| decimal_to_float32(&args[0], arg_type.clone(), tx)), - }, - }; - - Some(function) - }; - - registry.register_function_factory("to_float32", move |params, args_type| { - Some(Arc::new(factory(params, args_type)?)) - }); - registry.register_function_factory("to_float32", move |params, args_type| { - let f = factory(params, args_type)?; - Some(Arc::new(f.passthrough_nullable())) - }); - registry.register_function_factory("try_to_float32", move |params, args_type| { - let mut f = factory(params, args_type)?; - f.signature.name = "try_to_float32".to_string(); - Some(Arc::new(f.error_to_null())) - }); - registry.register_function_factory("try_to_float32", move |params, args_type| { - let mut f = factory(params, args_type)?; - f.signature.name = "try_to_float32".to_string(); - Some(Arc::new(f.error_to_null().passthrough_nullable())) - }); -} - -pub(crate) fn register_decimal_to_int(registry: &mut FunctionRegistry) { - if T::data_type().is_float() { - return; - } - let name = format!("to_{}", T::data_type().to_string().to_lowercase()); - let try_name = format!("try_to_{}", T::data_type().to_string().to_lowercase()); - - let factory = |_params: &[usize], args_type: &[DataType]| { - if args_type.len() != 1 { - return None; - } - - let name = format!("to_{}", T::data_type().to_string().to_lowercase()); - let arg_type = args_type[0].remove_nullable(); - if !arg_type.is_decimal() { - return None; - } - - let function = Function { - signature: FunctionSignature { - name, - args_type: vec![arg_type.clone()], - return_type: DataType::Number(T::data_type()), - }, - eval: FunctionEval::Scalar { - calc_domain: Box::new(|ctx, d| { - let res_fn = move || match d[0].as_decimal().unwrap() { - DecimalDomain::Decimal128(d, size) => Some(SimpleDomain:: { - min: d.min.to_int(size.scale, ctx.rounding_mode)?, - max: d.max.to_int(size.scale, ctx.rounding_mode)?, - }), - DecimalDomain::Decimal256(d, size) => Some(SimpleDomain:: { - min: d.min.to_int(size.scale, ctx.rounding_mode)?, - max: d.max.to_int(size.scale, ctx.rounding_mode)?, - }), - }; - - res_fn() - .map(|d| FunctionDomain::Domain(Domain::Number(T::upcast_domain(d)))) - .unwrap_or(FunctionDomain::MayThrow) - }), - eval: Box::new(move |args, tx| decimal_to_int::(&args[0], arg_type.clone(), tx)), - }, - }; - - Some(function) - }; - - registry.register_function_factory(&name, move |params, args_type| { - Some(Arc::new(factory(params, args_type)?)) - }); - registry.register_function_factory(&name, move |params, args_type| { - let f = factory(params, args_type)?; - Some(Arc::new(f.passthrough_nullable())) - }); - registry.register_function_factory(&try_name, move |params, args_type| { - let mut f = factory(params, args_type)?; - f.signature.name = format!("try_to_{}", T::data_type().to_string().to_lowercase()); - Some(Arc::new(f.error_to_null())) - }); - registry.register_function_factory(&try_name, move |params, args_type| { - let mut f = factory(params, args_type)?; - f.signature.name = format!("try_to_{}", T::data_type().to_string().to_lowercase()); - Some(Arc::new(f.error_to_null().passthrough_nullable())) - }); -} - -fn convert_to_decimal( - arg: &ValueRef, - ctx: &mut EvalContext, - from_type: &DataType, - dest_type: DecimalDataType, -) -> Value { - match from_type { - DataType::Boolean => boolean_to_decimal(arg, dest_type), - DataType::Number(ty) => { - if ty.is_float() { - float_to_decimal(arg, ctx, *ty, dest_type) - } else { - integer_to_decimal(arg, ctx, *ty, dest_type) - } - } - DataType::Decimal(from) => decimal_to_decimal(arg, ctx, *from, dest_type), - DataType::String => string_to_decimal(arg, ctx, dest_type), - _ => unreachable!("to_decimal not support this DataType"), - } -} - -fn convert_to_decimal_domain( - func_ctx: &FunctionContext, - domain: Domain, - dest_type: DecimalDataType, -) -> Option { - // Convert the domain to a Column. - // The first row is the min value, the second row is the max value. - let column = match domain { - Domain::Number(number_domain) => { - with_number_mapped_type!(|NUM_TYPE| match number_domain { - NumberDomain::NUM_TYPE(d) => { - let min = d.min; - let max = d.max; - NumberType::::from_data(vec![min, max]) - } - }) - } - Domain::Boolean(d) => { - let min = !d.has_false; - let max = d.has_true; - BooleanType::from_data(vec![min, max]) - } - Domain::Decimal(d) => { - with_decimal_mapped_type!(|DECIMAL| match d { - DecimalDomain::DECIMAL(d, size) => { - let min = d.min; - let max = d.max; - DecimalType::from_data_with_size(vec![min, max], size) - } - }) - } - Domain::String(d) => { - let min = d.min; - let max = d.max?; - StringType::from_data(vec![min, max]) - } - _ => { - return None; - } - }; - - let from_type = column.data_type(); - let value = Value::::Column(column); - let mut ctx = EvalContext { - generics: &[], - num_rows: 2, - func_ctx, - validity: None, - errors: None, - }; - let dest_size = dest_type.size(); - let res = convert_to_decimal(&value.as_ref(), &mut ctx, &from_type, dest_type); - - if ctx.errors.is_some() { - return None; - } - let decimal_col = res.as_column()?.as_decimal()?; - assert_eq!(decimal_col.len(), 2); - - Some(match decimal_col { - DecimalColumn::Decimal128(buf, size) => { - assert_eq!(&dest_size, size); - let (min, max) = unsafe { (*buf.get_unchecked(0), *buf.get_unchecked(1)) }; - DecimalDomain::Decimal128(SimpleDomain { min, max }, *size) - } - DecimalColumn::Decimal256(buf, size) => { - assert_eq!(&dest_size, size); - let (min, max) = unsafe { (*buf.get_unchecked(0), *buf.get_unchecked(1)) }; - DecimalDomain::Decimal256(SimpleDomain { min, max }, *size) - } - }) -} - -fn boolean_to_decimal_column( - boolean_column: &Bitmap, - size: DecimalSize, -) -> DecimalColumn { - let mut values = Vec::::with_capacity(boolean_column.len()); - for val in boolean_column.iter() { - if val { - values.push(T::e(size.scale as u32)); - } else { - values.push(T::zero()); - } - } - T::to_column(values, size) -} - -fn boolean_to_decimal_scalar(val: bool, size: DecimalSize) -> DecimalScalar { - if val { - T::to_scalar(T::e(size.scale as u32), size) - } else { - T::to_scalar(T::zero(), size) - } -} - -fn boolean_to_decimal(arg: &ValueRef, dest_type: DecimalDataType) -> Value { - match arg { - ValueRef::Column(column) => { - let boolean_column = BooleanType::try_downcast_column(column).unwrap(); - let column = match dest_type { - DecimalDataType::Decimal128(size) => { - boolean_to_decimal_column::(&boolean_column, size) - } - DecimalDataType::Decimal256(size) => { - boolean_to_decimal_column::(&boolean_column, size) - } - }; - Value::Column(Column::Decimal(column)) - } - ValueRef::Scalar(scalar) => { - let val = BooleanType::try_downcast_scalar(scalar).unwrap(); - let scalar = match dest_type { - DecimalDataType::Decimal128(size) => boolean_to_decimal_scalar::(val, size), - DecimalDataType::Decimal256(size) => boolean_to_decimal_scalar::(val, size), - }; - Value::Scalar(Scalar::Decimal(scalar)) - } - } -} - -fn string_to_decimal_column( - ctx: &mut EvalContext, - string_column: &StringColumn, - size: DecimalSize, - rounding_mode: bool, -) -> DecimalColumn { - let mut values = Vec::::with_capacity(string_column.len()); - for (row, buf) in string_column.iter().enumerate() { - match read_decimal_with_size::(buf, size, true, rounding_mode) { - Ok((d, _)) => values.push(d), - Err(e) => { - ctx.set_error(row, e.message()); - values.push(T::zero()) - } - } - } - T::to_column(values, size) -} - -fn string_to_decimal_scalar( - ctx: &mut EvalContext, - string_buf: &[u8], - size: DecimalSize, - rounding_mode: bool, -) -> DecimalScalar { - let value = match read_decimal_with_size::(string_buf, size, true, rounding_mode) { - Ok((d, _)) => d, - Err(e) => { - ctx.set_error(0, e.message()); - T::zero() - } - }; - T::to_scalar(value, size) -} - -fn string_to_decimal( - arg: &ValueRef, - ctx: &mut EvalContext, - dest_type: DecimalDataType, -) -> Value { - let rounding_mode = ctx.func_ctx.rounding_mode; - match arg { - ValueRef::Column(column) => { - let string_column = StringType::try_downcast_column(column).unwrap(); - let column = match dest_type { - DecimalDataType::Decimal128(size) => { - string_to_decimal_column::(ctx, &string_column, size, rounding_mode) - } - DecimalDataType::Decimal256(size) => { - string_to_decimal_column::(ctx, &string_column, size, rounding_mode) - } - }; - Value::Column(Column::Decimal(column)) - } - ValueRef::Scalar(scalar) => { - let buf = StringType::try_downcast_scalar(scalar).unwrap(); - let scalar = match dest_type { - DecimalDataType::Decimal128(size) => { - string_to_decimal_scalar::(ctx, buf, size, rounding_mode) - } - DecimalDataType::Decimal256(size) => { - string_to_decimal_scalar::(ctx, buf, size, rounding_mode) - } - }; - Value::Scalar(Scalar::Decimal(scalar)) - } - } -} - -fn integer_to_decimal( - arg: &ValueRef, - ctx: &mut EvalContext, - from_type: NumberDataType, - dest_type: DecimalDataType, -) -> Value { - let mut is_scalar = false; - let column = match arg { - ValueRef::Column(column) => column.clone(), - ValueRef::Scalar(s) => { - is_scalar = true; - let builder = ColumnBuilder::repeat(s, 1, &DataType::Number(from_type)); - builder.build() - } - }; - - let result = with_integer_mapped_type!(|NUM_TYPE| match from_type { - NumberDataType::NUM_TYPE => { - let column = NumberType::::try_downcast_column(&column).unwrap(); - integer_to_decimal_internal(column, ctx, &dest_type) - } - _ => unreachable!(), - }); - - if is_scalar { - let scalar = result.index(0).unwrap(); - Value::Scalar(Scalar::Decimal(scalar)) - } else { - Value::Column(Column::Decimal(result)) - } -} - -macro_rules! m_integer_to_decimal { - ($from: expr, $size: expr, $type_name: ty, $ctx: expr) => { - let multiplier = <$type_name>::e($size.scale as u32); - let min_for_precision = <$type_name>::min_for_precision($size.precision); - let max_for_precision = <$type_name>::max_for_precision($size.precision); - - let values = $from - .iter() - .enumerate() - .map(|(row, x)| { - let x = x.as_() * <$type_name>::one(); - let x = x.checked_mul(multiplier).and_then(|v| { - if v > max_for_precision || v < min_for_precision { - None - } else { - Some(v) - } - }); - - match x { - Some(x) => x, - None => { - $ctx.set_error(row, concat!("Decimal overflow at line : ", line!())); - <$type_name>::one() - } - } - }) - .collect(); - <$type_name>::to_column(values, $size) - }; -} - -fn integer_to_decimal_internal>( - from: Buffer, - ctx: &mut EvalContext, - dest_type: &DecimalDataType, -) -> DecimalColumn { - match dest_type { - DecimalDataType::Decimal128(size) => { - m_integer_to_decimal! {from, *size, i128, ctx} - } - DecimalDataType::Decimal256(size) => { - m_integer_to_decimal! {from, *size, i256, ctx} - } - } -} - -macro_rules! m_float_to_decimal { - ($from: expr, $size: expr, $type_name: ty, $ctx: expr) => { - let multiplier: f64 = (10_f64).powi($size.scale as i32).as_(); - - let min_for_precision = <$type_name>::min_for_precision($size.precision); - let max_for_precision = <$type_name>::max_for_precision($size.precision); - - let values = $from - .iter() - .enumerate() - .map(|(row, x)| { - let mut x = x.as_() * multiplier; - if $ctx.func_ctx.rounding_mode { - x = x.round(); - } - let x = <$type_name>::from_float(x); - if x > max_for_precision || x < min_for_precision { - $ctx.set_error(row, concat!("Decimal overflow at line : ", line!())); - <$type_name>::one() - } else { - x - } - }) - .collect(); - <$type_name>::to_column(values, $size) - }; -} - -fn float_to_decimal( - arg: &ValueRef, - ctx: &mut EvalContext, - from_type: NumberDataType, - dest_type: DecimalDataType, -) -> Value { - let mut is_scalar = false; - let column = match arg { - ValueRef::Column(column) => column.clone(), - ValueRef::Scalar(s) => { - is_scalar = true; - let builder = ColumnBuilder::repeat(s, 1, &DataType::Number(from_type)); - builder.build() - } - }; - - let result = match from_type { - NumberDataType::Float32 => { - let column = NumberType::::try_downcast_column(&column).unwrap(); - float_to_decimal_internal(column, ctx, &dest_type) - } - NumberDataType::Float64 => { - let column = NumberType::::try_downcast_column(&column).unwrap(); - float_to_decimal_internal(column, ctx, &dest_type) - } - _ => unreachable!(), - }; - if is_scalar { - let scalar = result.index(0).unwrap(); - Value::Scalar(Scalar::Decimal(scalar)) - } else { - Value::Column(Column::Decimal(result)) - } -} - -fn float_to_decimal_internal>( - from: Buffer, - ctx: &mut EvalContext, - dest_type: &DecimalDataType, -) -> DecimalColumn { - match dest_type { - DecimalDataType::Decimal128(size) => { - m_float_to_decimal! {from, *size, i128, ctx} - } - DecimalDataType::Decimal256(size) => { - m_float_to_decimal! {from, *size, i256, ctx} - } - } -} - -fn get_round_val(x: T, scale: u32, ctx: &mut EvalContext) -> Option { - let mut round_val = None; - if ctx.func_ctx.rounding_mode && scale > 0 { - // Checking whether numbers need to be added or subtracted to calculate rounding - if let Some(r) = x.checked_rem(T::e(scale)) { - if let Some(m) = r.checked_div(T::e(scale - 1)) { - if m >= T::from_i64(5i64) { - round_val = Some(T::one()); - } else if m <= T::from_i64(-5i64) { - round_val = Some(T::minus_one()); - } - } - } - } - round_val -} - -fn decimal_256_to_128( - buffer: Buffer, - from_size: DecimalSize, - dest_size: DecimalSize, - ctx: &mut EvalContext, -) -> DecimalColumn { - let max = i128::max_for_precision(dest_size.precision); - let min = i128::min_for_precision(dest_size.precision); - - let values = if dest_size.scale >= from_size.scale { - let factor = i256::e((dest_size.scale - from_size.scale) as u32); - buffer - .iter() - .enumerate() - .map(|(row, x)| { - let x = x * i128::one(); - match x.checked_mul(factor) { - Some(x) if x <= max && x >= min => *x.low(), - _ => { - ctx.set_error(row, concat!("Decimal overflow at line : ", line!())); - i128::one() - } - } - }) - .collect() - } else { - let scale_diff = (from_size.scale - dest_size.scale) as u32; - let factor = i256::e(scale_diff); - let source_factor = i256::e(from_size.scale as u32); - - buffer - .iter() - .enumerate() - .map(|(row, x)| { - let x = x * i128::one(); - let round_val = get_round_val::(x, scale_diff, ctx); - let y = match (x.checked_div(factor), round_val) { - (Some(x), Some(round_val)) => x.checked_add(round_val), - (Some(x), None) => Some(x), - (None, _) => None, - }; - - match y { - Some(y) if (y <= max && y >= min) && (y != 0 || x / source_factor == 0) => { - *y.low() - } - _ => { - ctx.set_error(row, concat!("Decimal overflow at line : ", line!())); - i128::one() - } - } - }) - .collect() - }; - i128::to_column(values, dest_size) -} - -macro_rules! m_decimal_to_decimal { - ($from_size: expr, $dest_size: expr, $buffer: expr, $from_type_name: ty, $dest_type_name: ty, $ctx: expr) => { - // faster path - if $from_size.scale == $dest_size.scale && $from_size.precision <= $dest_size.precision { - if <$from_type_name>::MAX == <$dest_type_name>::MAX { - // 128 -> 128 or 256 -> 256 - <$from_type_name>::to_column_from_buffer($buffer, $dest_size) - } else { - // 128 -> 256 - let buffer = $buffer - .into_iter() - .map(|x| x * <$dest_type_name>::one()) - .collect(); - <$dest_type_name>::to_column(buffer, $dest_size) - } - } else { - let values: Vec<_> = if $from_size.scale > $dest_size.scale { - let scale_diff = ($from_size.scale - $dest_size.scale) as u32; - let factor = <$dest_type_name>::e(scale_diff); - let max = <$dest_type_name>::max_for_precision($dest_size.precision); - let min = <$dest_type_name>::min_for_precision($dest_size.precision); - - let source_factor = <$from_type_name>::e($from_size.scale as u32); - $buffer - .iter() - .enumerate() - .map(|(row, x)| { - let x = x * <$dest_type_name>::one(); - let round_val = get_round_val::<$dest_type_name>(x, scale_diff, $ctx); - let y = match (x.checked_div(factor), round_val) { - (Some(x), Some(round_val)) => x.checked_add(round_val), - (Some(x), None) => Some(x), - (None, _) => None, - }; - match y { - Some(y) - if y <= max && y >= min && (y != 0 || x / source_factor == 0) => - { - y as $dest_type_name - } - _ => { - $ctx.set_error( - row, - concat!("Decimal overflow at line : ", line!()), - ); - <$dest_type_name>::one() - } - } - }) - .collect() - } else { - let factor = <$dest_type_name>::e(($dest_size.scale - $from_size.scale) as u32); - let max = <$dest_type_name>::max_for_precision($dest_size.precision); - let min = <$dest_type_name>::min_for_precision($dest_size.precision); - $buffer - .iter() - .enumerate() - .map(|(row, x)| { - let x = x * <$dest_type_name>::one(); - match x.checked_mul(factor) { - Some(x) if x <= max && x >= min => x as $dest_type_name, - _ => { - $ctx.set_error( - row, - concat!("Decimal overflow at line : ", line!()), - ); - <$dest_type_name>::one() - } - } - }) - .collect() - }; - <$dest_type_name>::to_column(values, $dest_size) - } - }; -} - -fn decimal_to_decimal( - arg: &ValueRef, - ctx: &mut EvalContext, - from_type: DecimalDataType, - dest_type: DecimalDataType, -) -> Value { - let mut is_scalar = false; - let column = match arg { - ValueRef::Column(column) => column.clone(), - ValueRef::Scalar(s) => { - is_scalar = true; - let builder = ColumnBuilder::repeat(s, 1, &DataType::Decimal(from_type)); - builder.build() - } - }; - - let result: DecimalColumn = match (from_type, dest_type) { - (DecimalDataType::Decimal128(_), DecimalDataType::Decimal128(dest_size)) => { - let (buffer, from_size) = i128::try_downcast_column(&column).unwrap(); - m_decimal_to_decimal! {from_size, dest_size, buffer, i128, i128, ctx} - } - (DecimalDataType::Decimal128(_), DecimalDataType::Decimal256(dest_size)) => { - let (buffer, from_size) = i128::try_downcast_column(&column).unwrap(); - m_decimal_to_decimal! {from_size, dest_size, buffer, i128, i256, ctx} - } - (DecimalDataType::Decimal256(_), DecimalDataType::Decimal256(dest_size)) => { - let (buffer, from_size) = i256::try_downcast_column(&column).unwrap(); - m_decimal_to_decimal! {from_size, dest_size, buffer, i256, i256, ctx} - } - (DecimalDataType::Decimal256(_), DecimalDataType::Decimal128(dest_size)) => { - let (buffer, from_size) = i256::try_downcast_column(&column).unwrap(); - decimal_256_to_128(buffer, from_size, dest_size, ctx) - } - }; - - if is_scalar { - let scalar = result.index(0).unwrap(); - Value::Scalar(Scalar::Decimal(scalar)) - } else { - Value::Column(Column::Decimal(result)) - } -} - -fn decimal_to_float64( - arg: &ValueRef, - from_type: DataType, - _ctx: &mut EvalContext, -) -> Value { - let mut is_scalar = false; - let column = match arg { - ValueRef::Column(column) => column.clone(), - ValueRef::Scalar(s) => { - is_scalar = true; - let builder = ColumnBuilder::repeat(s, 1, &from_type); - builder.build() - } - }; - - let from_type = from_type.as_decimal().unwrap(); - - let result = match from_type { - DecimalDataType::Decimal128(_) => { - let (buffer, from_size) = i128::try_downcast_column(&column).unwrap(); - - let div = 10_f64.powi(from_size.scale as i32); - - let values: Buffer = buffer.iter().map(|x| (*x as f64 / div).into()).collect(); - Float64Type::upcast_column(values) - } - - DecimalDataType::Decimal256(_) => { - let (buffer, from_size) = i256::try_downcast_column(&column).unwrap(); - - let div = 10_f64.powi(from_size.scale as i32); - - let values: Buffer = buffer - .iter() - .map(|x| (f64::from(*x) / div).into()) - .collect(); - Float64Type::upcast_column(values) - } - }; - - if is_scalar { - let scalar = result.index(0).unwrap(); - Value::Scalar(scalar.to_owned()) - } else { - Value::Column(result) - } -} - -fn decimal_to_float32( - arg: &ValueRef, - from_type: DataType, - _ctx: &mut EvalContext, -) -> Value { - let mut is_scalar = false; - let column = match arg { - ValueRef::Column(column) => column.clone(), - ValueRef::Scalar(s) => { - is_scalar = true; - let builder = ColumnBuilder::repeat(s, 1, &from_type); - builder.build() - } - }; - - let from_type = from_type.as_decimal().unwrap(); - - let result = match from_type { - DecimalDataType::Decimal128(_) => { - let (buffer, from_size) = i128::try_downcast_column(&column).unwrap(); - - let div = 10_f32.powi(from_size.scale as i32); - - let values: Buffer = buffer.iter().map(|x| (*x as f32 / div).into()).collect(); - Float32Type::upcast_column(values) - } - - DecimalDataType::Decimal256(_) => { - let (buffer, from_size) = i256::try_downcast_column(&column).unwrap(); - - let div = 10_f32.powi(from_size.scale as i32); - - let values: Buffer = buffer - .iter() - .map(|x| (f32::from(*x) / div).into()) - .collect(); - Float32Type::upcast_column(values) - } - }; - - if is_scalar { - let scalar = result.index(0).unwrap(); - Value::Scalar(scalar.to_owned()) - } else { - Value::Column(result) - } -} - -fn decimal_to_int( - arg: &ValueRef, - from_type: DataType, - ctx: &mut EvalContext, -) -> Value { - let mut is_scalar = false; - let column = match arg { - ValueRef::Column(column) => column.clone(), - ValueRef::Scalar(s) => { - is_scalar = true; - let builder = ColumnBuilder::repeat(s, 1, &from_type); - builder.build() - } - }; - - let from_type = from_type.as_decimal().unwrap(); - - let result = match from_type { - DecimalDataType::Decimal128(_) => { - let (buffer, from_size) = i128::try_downcast_column(&column).unwrap(); - - let mut values = Vec::with_capacity(ctx.num_rows); - - for (i, x) in buffer.iter().enumerate() { - match x.to_int(from_size.scale, ctx.func_ctx.rounding_mode) { - Some(x) => values.push(x), - None => { - ctx.set_error(i, "decimal cast to int overflow"); - values.push(T::default()) - } - } - } - - NumberType::::upcast_column(Buffer::from(values)) - } - - DecimalDataType::Decimal256(_) => { - let (buffer, from_size) = i256::try_downcast_column(&column).unwrap(); - let mut values = Vec::with_capacity(ctx.num_rows); - - for (i, x) in buffer.iter().enumerate() { - match x.to_int(from_size.scale, ctx.func_ctx.rounding_mode) { - Some(x) => values.push(x), - None => { - ctx.set_error(i, "decimal cast to int overflow"); - values.push(T::default()) - } - } - } - NumberType::::upcast_column(Buffer::from(values)) - } - }; - - if is_scalar { - let scalar = result.index(0).unwrap(); - Value::Scalar(scalar.to_owned()) - } else { - Value::Column(result) - } -} - -pub fn register_decimal_math(registry: &mut FunctionRegistry) { - let factory = |params: &[usize], args_type: &[DataType], round_mode: RoundMode| { - if args_type.is_empty() { - return None; - } - - let from_type = args_type[0].remove_nullable(); - if !matches!(from_type, DataType::Decimal(_)) { - return None; - } - - let from_decimal_type = from_type.as_decimal().unwrap(); - - let scale = if params.is_empty() { - 0 - } else { - params[0] as i64 - 76 - }; - - let decimal_size = DecimalSize { - precision: from_decimal_type.precision(), - scale: scale.clamp(0, from_decimal_type.scale() as i64) as u8, - }; - - let dest_decimal_type = DecimalDataType::from_size(decimal_size).ok()?; - let name = format!("{:?}", round_mode).to_lowercase(); - - let mut sig_args_type = args_type.to_owned(); - sig_args_type[0] = from_type.clone(); - let f = Function { - signature: FunctionSignature { - name, - args_type: sig_args_type, - return_type: DataType::Decimal(dest_decimal_type), - }, - eval: FunctionEval::Scalar { - calc_domain: Box::new(move |_ctx, _d| FunctionDomain::Full), - eval: Box::new(move |args, _ctx| { - decimal_round_truncate( - &args[0], - from_type.clone(), - dest_decimal_type, - scale, - round_mode, - ) - }), - }, - }; - - if args_type[0].is_nullable() { - Some(f.passthrough_nullable()) - } else { - Some(f) - } - }; - - for m in [ - RoundMode::Round, - RoundMode::Truncate, - RoundMode::Ceil, - RoundMode::Floor, - ] { - let name = format!("{:?}", m).to_lowercase(); - registry.register_function_factory(&name, move |params, args_type| { - Some(Arc::new(factory(params, args_type, m)?)) - }); - } -} - -#[derive(Copy, Clone, Debug)] -enum RoundMode { - Round, - Truncate, - Floor, - Ceil, -} - -fn decimal_round_positive(values: &[T], source_scale: i64, target_scale: i64) -> Vec -where T: Decimal + From + DivAssign + Div + Add + Sub { - let power_of_ten = T::e((source_scale - target_scale) as u32); - let addition = power_of_ten / T::from(2); - - values - .iter() - .map(|input| { - let input = if input < &T::zero() { - *input - addition - } else { - *input + addition - }; - input / power_of_ten - }) - .collect() -} - -fn decimal_round_negative(values: &[T], source_scale: i64, target_scale: i64) -> Vec -where T: Decimal - + From - + DivAssign - + Div - + Add - + Sub - + Mul { - let divide_power_of_ten = T::e((source_scale - target_scale) as u32); - let addition = divide_power_of_ten / T::from(2); - let multiply_power_of_ten = T::e((-target_scale) as u32); - - values - .iter() - .map(|input| { - let input = if input < &T::zero() { - *input - addition - } else { - *input + addition - }; - input / divide_power_of_ten * multiply_power_of_ten - }) - .collect() -} - -// if round mode is ceil, truncate should add one value -fn decimal_truncate_positive(values: &[T], source_scale: i64, target_scale: i64) -> Vec -where T: Decimal + From + DivAssign + Div + Add + Sub { - let power_of_ten = T::e((source_scale - target_scale) as u32); - - values.iter().map(|input| *input / power_of_ten).collect() -} - -fn decimal_truncate_negative(values: &[T], source_scale: i64, target_scale: i64) -> Vec -where T: Decimal - + From - + DivAssign - + Div - + Add - + Sub - + Mul { - let divide_power_of_ten = T::e((source_scale - target_scale) as u32); - let multiply_power_of_ten = T::e((-target_scale) as u32); - - values - .iter() - .map(|input| *input / divide_power_of_ten * multiply_power_of_ten) - .collect() -} - -fn decimal_floor(values: &[T], source_scale: i64) -> Vec -where T: Decimal - + From - + DivAssign - + Div - + Add - + Sub - + Mul { - let power_of_ten = T::e(source_scale as u32); - - values - .iter() - .map(|input| { - if input < &T::zero() { - // below 0 we ceil the number (e.g. -10.5 -> -11) - ((*input + T::one()) / power_of_ten) - T::one() - } else { - *input / power_of_ten - } - }) - .collect() -} - -fn decimal_ceil(values: &[T], source_scale: i64) -> Vec -where T: Decimal - + From - + DivAssign - + Div - + Add - + Sub - + Mul { - let power_of_ten = T::e(source_scale as u32); - - values - .iter() - .map(|input| { - if input <= &T::zero() { - *input / power_of_ten - } else { - ((*input - T::one()) / power_of_ten) + T::one() - } - }) - .collect() -} - -fn decimal_round_truncate( - arg: &ValueRef, - from_type: DataType, - dest_type: DecimalDataType, - target_scale: i64, - mode: RoundMode, -) -> Value { - let from_decimal_type = from_type.as_decimal().unwrap(); - let source_scale = from_decimal_type.scale() as i64; - - if source_scale < target_scale { - return arg.clone().to_owned(); - } - - let mut is_scalar = false; - let column = match arg { - ValueRef::Column(column) => column.clone(), - ValueRef::Scalar(s) => { - is_scalar = true; - let builder = ColumnBuilder::repeat(s, 1, &from_type); - builder.build() - } - }; - - let none_negative = target_scale >= 0; - - let result = match from_decimal_type { - DecimalDataType::Decimal128(_) => { - let (buffer, _) = i128::try_downcast_column(&column).unwrap(); - - let result = match (none_negative, mode) { - (true, RoundMode::Round) => { - decimal_round_positive::<_>(&buffer, source_scale, target_scale) - } - (true, RoundMode::Truncate) => { - decimal_truncate_positive::<_>(&buffer, source_scale, target_scale) - } - (false, RoundMode::Round) => { - decimal_round_negative::<_>(&buffer, source_scale, target_scale) - } - (false, RoundMode::Truncate) => { - decimal_truncate_negative::<_>(&buffer, source_scale, target_scale) - } - (_, RoundMode::Floor) => decimal_floor::<_>(&buffer, source_scale), - (_, RoundMode::Ceil) => decimal_ceil::<_>(&buffer, source_scale), - }; - i128::to_column(result, dest_type.size()) - } - - DecimalDataType::Decimal256(_) => { - let (buffer, _) = i256::try_downcast_column(&column).unwrap(); - let result = match (none_negative, mode) { - (true, RoundMode::Round) => { - decimal_round_positive::<_>(&buffer, source_scale, target_scale) - } - (true, RoundMode::Truncate) => { - decimal_truncate_positive::<_>(&buffer, source_scale, target_scale) - } - (false, RoundMode::Round) => { - decimal_round_negative::<_>(&buffer, source_scale, target_scale) - } - (false, RoundMode::Truncate) => { - decimal_truncate_negative::<_>(&buffer, source_scale, target_scale) - } - (_, RoundMode::Floor) => decimal_floor::<_>(&buffer, source_scale), - (_, RoundMode::Ceil) => decimal_ceil::<_>(&buffer, source_scale), - }; - i256::to_column(result, dest_type.size()) - } - }; - - let result = Column::Decimal(result); - if is_scalar { - let scalar = result.index(0).unwrap(); - Value::Scalar(scalar.to_owned()) - } else { - Value::Column(result) - } -} diff --git a/src/query/functions/src/scalars/decimal/arithmetic.rs b/src/query/functions/src/scalars/decimal/arithmetic.rs new file mode 100644 index 000000000000..dec2bfecb87f --- /dev/null +++ b/src/query/functions/src/scalars/decimal/arithmetic.rs @@ -0,0 +1,363 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::ops::*; +use std::sync::Arc; + +use databend_common_expression::types::decimal::*; +use databend_common_expression::types::*; +use databend_common_expression::vectorize_2_arg; +use databend_common_expression::vectorize_with_builder_2_arg; +use databend_common_expression::Domain; +use databend_common_expression::EvalContext; +use databend_common_expression::Function; +use databend_common_expression::FunctionDomain; +use databend_common_expression::FunctionEval; +use databend_common_expression::FunctionRegistry; +use databend_common_expression::FunctionSignature; +use ethnum::i256; + +#[derive(Copy, Clone, Debug)] +enum ArithmeticOp { + Plus, + Minus, + Multiply, + Divide, +} + +macro_rules! op_decimal { + ($a: expr, $b: expr, $ctx: expr, $left: expr, $right: expr, $result_type: expr, $op: ident, $arithmetic_op: expr) => { + match $left { + DecimalDataType::Decimal128(_) => { + binary_decimal!( + $a, + $b, + $ctx, + $left, + $right, + $op, + $result_type.size(), + i128, + $arithmetic_op + ) + } + DecimalDataType::Decimal256(_) => { + binary_decimal!( + $a, + $b, + $ctx, + $left, + $right, + $op, + $result_type.size(), + i256, + $arithmetic_op + ) + } + } + }; +} + +macro_rules! binary_decimal { + ($a: expr, $b: expr, $ctx: expr, $left: expr, $right: expr, $op: ident, $size: expr, $type_name: ty, $arithmetic_op: expr) => {{ + type T = $type_name; + + let overflow = $size.precision == T::default_decimal_size().precision; + + let a = $a.try_downcast().unwrap(); + let b = $b.try_downcast().unwrap(); + + let zero = T::zero(); + let one = T::one(); + + let result = if matches!($arithmetic_op, ArithmeticOp::Divide) { + let scale_a = $left.scale(); + let scale_b = $right.scale(); + + let (scale_mul, scale_div) = if scale_b + $size.scale > scale_a { + (scale_b + $size.scale - scale_a, 0) + } else { + (0, scale_b + $size.scale - scale_a) + }; + + let multiplier = T::e(scale_mul as u32); + let div = T::e(scale_div as u32); + + let func = |a: T, b: T, result: &mut Vec, ctx: &mut EvalContext| { + if std::intrinsics::unlikely(b == zero) { + ctx.set_error(result.len(), "divided by zero"); + result.push(one); + } else { + result.push((a * multiplier).div(b) / div); + } + }; + + vectorize_with_builder_2_arg::, DecimalType, DecimalType>(func)( + a, b, $ctx, + ) + } else { + if overflow { + let min_for_precision = T::min_for_precision($size.precision); + let max_for_precision = T::max_for_precision($size.precision); + + let func = |a: T, b: T, result: &mut Vec, ctx: &mut EvalContext| { + let t = a.$op(b); + if t < min_for_precision || t > max_for_precision { + ctx.set_error( + result.len(), + concat!("Decimal overflow at line : ", line!()), + ); + result.push(one); + } else { + result.push(t); + } + }; + + vectorize_with_builder_2_arg::, DecimalType, DecimalType>(func)( + a, b, $ctx + ) + } else { + let func = |l: T, r: T, _ctx: &mut EvalContext| l.$op(r); + + vectorize_2_arg::, DecimalType, DecimalType>(func)( + a, b, $ctx + ) + } + }; + result.upcast_decimal($size) + }}; +} + +#[inline(always)] +fn domain_plus( + lhs: &SimpleDomain, + rhs: &SimpleDomain, + precision: u8, +) -> Option> { + // For plus, the scale of the two operands must be the same. + let min = T::min_for_precision(precision); + let max = T::max_for_precision(precision); + Some(SimpleDomain { + min: lhs + .min + .checked_add(rhs.min) + .filter(|&m| m >= min && m <= max)?, + max: lhs + .max + .checked_add(rhs.max) + .filter(|&m| m >= min && m <= max)?, + }) +} + +#[inline(always)] +fn domain_minus( + lhs: &SimpleDomain, + rhs: &SimpleDomain, + precision: u8, +) -> Option> { + // For minus, the scale of the two operands must be the same. + let min = T::min_for_precision(precision); + let max = T::max_for_precision(precision); + Some(SimpleDomain { + min: lhs + .min + .checked_sub(rhs.max) + .filter(|&m| m >= min && m <= max)?, + max: lhs + .max + .checked_sub(rhs.min) + .filter(|&m| m >= min && m <= max)?, + }) +} + +#[inline(always)] +fn domain_mul( + lhs: &SimpleDomain, + rhs: &SimpleDomain, + precision: u8, +) -> Option> { + let min = T::min_for_precision(precision); + let max = T::max_for_precision(precision); + + let a = lhs + .min + .checked_mul(rhs.min) + .filter(|&m| m >= min && m <= max)?; + let b = lhs + .min + .checked_mul(rhs.max) + .filter(|&m| m >= min && m <= max)?; + let c = lhs + .max + .checked_mul(rhs.min) + .filter(|&m| m >= min && m <= max)?; + let d = lhs + .max + .checked_mul(rhs.max) + .filter(|&m| m >= min && m <= max)?; + + Some(SimpleDomain { + min: a.min(b).min(c).min(d), + max: a.max(b).max(c).max(d), + }) +} + +#[inline(always)] +fn domain_div( + _lhs: &SimpleDomain, + _rhs: &SimpleDomain, + _precision: u8, +) -> Option> { + // For div, we cannot determine the domain. + None +} + +macro_rules! register_decimal_binary_op { + ($registry: expr, $arithmetic_op: expr, $op: ident, $domain_op: ident, $default_domain: expr) => { + let name = format!("{:?}", $arithmetic_op).to_lowercase(); + + $registry.register_function_factory(&name, |_, args_type| { + if args_type.len() != 2 { + return None; + } + + let has_nullable = args_type.iter().any(|x| x.is_nullable_or_null()); + let args_type: Vec = args_type.iter().map(|x| x.remove_nullable()).collect(); + + // number X decimal -> decimal + // decimal X number -> decimal + // decimal X decimal -> decimal + if !args_type[0].is_decimal() && !args_type[1].is_decimal() { + return None; + } + + let decimal_a = + DecimalDataType::from_size(args_type[0].get_decimal_properties()?).unwrap(); + let decimal_b = + DecimalDataType::from_size(args_type[1].get_decimal_properties()?).unwrap(); + + let is_multiply = matches!($arithmetic_op, ArithmeticOp::Multiply); + let is_divide = matches!($arithmetic_op, ArithmeticOp::Divide); + let is_plus_minus = !is_multiply && !is_divide; + + // left, right will unify to same width decimal, both 256 or both 128 + let (left, right, return_decimal_type) = DecimalDataType::binary_result_type( + &decimal_a, + &decimal_b, + is_multiply, + is_divide, + is_plus_minus, + ) + .ok()?; + + let function = Function { + signature: FunctionSignature { + name: format!("{:?}", $arithmetic_op).to_lowercase(), + args_type: vec![ + DataType::Decimal(left.clone()), + DataType::Decimal(right.clone()), + ], + return_type: DataType::Decimal(return_decimal_type), + }, + eval: FunctionEval::Scalar { + calc_domain: Box::new(move |_ctx, d| { + let lhs = d[0].as_decimal(); + let rhs = d[1].as_decimal(); + + if lhs.is_none() || rhs.is_none() { + return FunctionDomain::Full; + } + + let lhs = lhs.unwrap(); + let rhs = rhs.unwrap(); + + let size = return_decimal_type.size(); + + { + match (lhs, rhs) { + ( + DecimalDomain::Decimal128(d1, _), + DecimalDomain::Decimal128(d2, _), + ) => $domain_op(&d1, &d2, size.precision) + .map(|d| DecimalDomain::Decimal128(d, size)), + ( + DecimalDomain::Decimal256(d1, _), + DecimalDomain::Decimal256(d2, _), + ) => $domain_op(&d1, &d2, size.precision) + .map(|d| DecimalDomain::Decimal256(d, size)), + _ => { + unreachable!("unreachable decimal domain {:?} /{:?}", lhs, rhs) + } + } + } + .map(|d| FunctionDomain::Domain(Domain::Decimal(d))) + .unwrap_or($default_domain) + }), + eval: Box::new(move |args, ctx| { + let res = op_decimal!( + &args[0], + &args[1], + ctx, + left, + right, + return_decimal_type, + $op, + $arithmetic_op + ); + + res + }), + }, + }; + if has_nullable { + Some(Arc::new(function.passthrough_nullable())) + } else { + Some(Arc::new(function)) + } + }); + }; +} + +pub(crate) fn register_decimal_arithmetic(registry: &mut FunctionRegistry) { + // TODO checked overflow by default + register_decimal_binary_op!( + registry, + ArithmeticOp::Plus, + add, + domain_plus, + FunctionDomain::Full + ); + + register_decimal_binary_op!( + registry, + ArithmeticOp::Minus, + sub, + domain_minus, + FunctionDomain::Full + ); + register_decimal_binary_op!( + registry, + ArithmeticOp::Divide, + div, + domain_div, + FunctionDomain::MayThrow + ); + register_decimal_binary_op!( + registry, + ArithmeticOp::Multiply, + mul, + domain_mul, + FunctionDomain::Full + ); +} diff --git a/src/query/functions/src/scalars/decimal/cast.rs b/src/query/functions/src/scalars/decimal/cast.rs new file mode 100644 index 000000000000..58b3d9d83d60 --- /dev/null +++ b/src/query/functions/src/scalars/decimal/cast.rs @@ -0,0 +1,746 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::ops::Mul; +use std::sync::Arc; + +use databend_common_expression::serialize::read_decimal_with_size; +use databend_common_expression::types::decimal::*; +use databend_common_expression::types::*; +use databend_common_expression::vectorize_1_arg; +use databend_common_expression::vectorize_with_builder_1_arg; +use databend_common_expression::with_decimal_mapped_type; +use databend_common_expression::with_integer_mapped_type; +use databend_common_expression::with_number_mapped_type; +use databend_common_expression::Domain; +use databend_common_expression::EvalContext; +use databend_common_expression::FromData; +use databend_common_expression::Function; +use databend_common_expression::FunctionContext; +use databend_common_expression::FunctionDomain; +use databend_common_expression::FunctionEval; +use databend_common_expression::FunctionRegistry; +use databend_common_expression::FunctionSignature; +use databend_common_expression::Value; +use databend_common_expression::ValueRef; +use ethnum::i256; +use num_traits::AsPrimitive; +use ordered_float::OrderedFloat; + +// int float to decimal +pub fn register_to_decimal(registry: &mut FunctionRegistry) { + let factory = |params: &[usize], args_type: &[DataType]| { + if args_type.len() != 1 { + return None; + } + if params.len() != 2 { + return None; + } + + let from_type = args_type[0].remove_nullable(); + + if !matches!( + from_type, + DataType::Boolean | DataType::Number(_) | DataType::Decimal(_) | DataType::String + ) { + return None; + } + + let decimal_size = DecimalSize { + precision: params[0] as u8, + scale: params[1] as u8, + }; + + let decimal_type = DecimalDataType::from_size(decimal_size).ok()?; + + Some(Function { + signature: FunctionSignature { + name: "to_decimal".to_string(), + args_type: vec![from_type.clone()], + return_type: DataType::Decimal(decimal_type), + }, + eval: FunctionEval::Scalar { + calc_domain: Box::new(move |ctx, d| { + convert_to_decimal_domain(ctx, d[0].clone(), decimal_type) + .map(|d| FunctionDomain::Domain(Domain::Decimal(d))) + .unwrap_or(FunctionDomain::MayThrow) + }), + eval: Box::new(move |args, ctx| { + convert_to_decimal(&args[0], ctx, &from_type, decimal_type) + }), + }, + }) + }; + + registry.register_function_factory("to_decimal", move |params, args_type| { + Some(Arc::new(factory(params, args_type)?)) + }); + registry.register_function_factory("to_decimal", move |params, args_type| { + let f = factory(params, args_type)?; + Some(Arc::new(f.passthrough_nullable())) + }); + registry.register_function_factory("try_to_decimal", move |params, args_type| { + let mut f = factory(params, args_type)?; + f.signature.name = "try_to_decimal".to_string(); + Some(Arc::new(f.error_to_null())) + }); + registry.register_function_factory("try_to_decimal", move |params, args_type| { + let mut f = factory(params, args_type)?; + f.signature.name = "try_to_decimal".to_string(); + Some(Arc::new(f.error_to_null().passthrough_nullable())) + }); +} + +pub(crate) fn register_decimal_to_float(registry: &mut FunctionRegistry) { + let data_type = NumberType::::data_type(); + debug_assert!(data_type.is_floating()); + + let is_f32 = matches!(data_type, DataType::Number(NumberDataType::Float32)); + + let factory = |_params: &[usize], args_type: &[DataType], data_type: DataType| { + if args_type.len() != 1 { + return None; + } + + let arg_type = args_type[0].remove_nullable(); + if !arg_type.is_decimal() { + return None; + } + let is_f32 = matches!(data_type, DataType::Number(NumberDataType::Float32)); + let name = if is_f32 { "to_float32" } else { "to_float64" }; + let calc_domain = if is_f32 { + Box::new(|_: &_, d: &[Domain]| { + with_decimal_mapped_type!(|DECIMAL_TYPE| match d[0].as_decimal().unwrap() { + DecimalDomain::DECIMAL_TYPE(d, size) => { + FunctionDomain::Domain(Domain::Number(NumberDomain::Float32( + SimpleDomain { + min: OrderedFloat(d.min.to_float32(size.scale)), + max: OrderedFloat(d.max.to_float32(size.scale)), + }, + ))) + } + }) + }) as _ + } else { + Box::new(|_: &_, d: &[Domain]| { + with_decimal_mapped_type!(|DECIMAL_TYPE| match d[0].as_decimal().unwrap() { + DecimalDomain::DECIMAL_TYPE(d, size) => { + FunctionDomain::Domain(Domain::Number(NumberDomain::Float64( + SimpleDomain { + min: OrderedFloat(d.min.to_float64(size.scale)), + max: OrderedFloat(d.max.to_float64(size.scale)), + }, + ))) + } + }) + }) as _ + }; + + let eval = if is_f32 { + let arg_type = arg_type.clone(); + Box::new(move |args: &[ValueRef], tx: &mut EvalContext| { + decimal_to_float::(&args[0], arg_type.clone(), tx) + }) as _ + } else { + let arg_type = arg_type.clone(); + Box::new(move |args: &[ValueRef], tx: &mut EvalContext| { + decimal_to_float::(&args[0], arg_type.clone(), tx) + }) as _ + }; + + let function = Function { + signature: FunctionSignature { + name: name.to_string(), + args_type: vec![arg_type.clone()], + return_type: data_type.clone(), + }, + eval: FunctionEval::Scalar { calc_domain, eval }, + }; + + Some(function) + }; + + let name = if is_f32 { "to_float32" } else { "to_float64" }; + + registry.register_function_factory(name, move |params, args_type| { + let data_type = NumberType::::data_type(); + Some(Arc::new(factory(params, args_type, data_type)?)) + }); + registry.register_function_factory(name, move |params, args_type| { + let data_type = NumberType::::data_type(); + let f = factory(params, args_type, data_type)?; + Some(Arc::new(f.passthrough_nullable())) + }); + registry.register_function_factory(&format!("try_{name}"), move |params, args_type| { + let data_type = NumberType::::data_type(); + let mut f = factory(params, args_type, data_type)?; + f.signature.name = format!("try_{name}"); + Some(Arc::new(f.error_to_null())) + }); + registry.register_function_factory(&format!("try_{name}"), move |params, args_type| { + let data_type = NumberType::::data_type(); + let mut f = factory(params, args_type, data_type)?; + f.signature.name = format!("try_{name}"); + Some(Arc::new(f.error_to_null().passthrough_nullable())) + }); +} + +pub(crate) fn register_decimal_to_int(registry: &mut FunctionRegistry) { + if T::data_type().is_float() { + return; + } + let name = format!("to_{}", T::data_type().to_string().to_lowercase()); + let try_name = format!("try_to_{}", T::data_type().to_string().to_lowercase()); + + let factory = |_params: &[usize], args_type: &[DataType]| { + if args_type.len() != 1 { + return None; + } + + let name = format!("to_{}", T::data_type().to_string().to_lowercase()); + let arg_type = args_type[0].remove_nullable(); + if !arg_type.is_decimal() { + return None; + } + + let function = Function { + signature: FunctionSignature { + name, + args_type: vec![arg_type.clone()], + return_type: DataType::Number(T::data_type()), + }, + eval: FunctionEval::Scalar { + calc_domain: Box::new(|ctx, d| { + let res_fn = move || match d[0].as_decimal().unwrap() { + DecimalDomain::Decimal128(d, size) => Some(SimpleDomain:: { + min: d.min.to_int(size.scale, ctx.rounding_mode)?, + max: d.max.to_int(size.scale, ctx.rounding_mode)?, + }), + DecimalDomain::Decimal256(d, size) => Some(SimpleDomain:: { + min: d.min.to_int(size.scale, ctx.rounding_mode)?, + max: d.max.to_int(size.scale, ctx.rounding_mode)?, + }), + }; + + res_fn() + .map(|d| FunctionDomain::Domain(Domain::Number(T::upcast_domain(d)))) + .unwrap_or(FunctionDomain::MayThrow) + }), + eval: Box::new(move |args, tx| decimal_to_int::(&args[0], arg_type.clone(), tx)), + }, + }; + + Some(function) + }; + + registry.register_function_factory(&name, move |params, args_type| { + Some(Arc::new(factory(params, args_type)?)) + }); + registry.register_function_factory(&name, move |params, args_type| { + let f = factory(params, args_type)?; + Some(Arc::new(f.passthrough_nullable())) + }); + registry.register_function_factory(&try_name, move |params, args_type| { + let mut f = factory(params, args_type)?; + f.signature.name = format!("try_to_{}", T::data_type().to_string().to_lowercase()); + Some(Arc::new(f.error_to_null())) + }); + registry.register_function_factory(&try_name, move |params, args_type| { + let mut f = factory(params, args_type)?; + f.signature.name = format!("try_to_{}", T::data_type().to_string().to_lowercase()); + Some(Arc::new(f.error_to_null().passthrough_nullable())) + }); +} + +fn convert_to_decimal( + arg: &ValueRef, + ctx: &mut EvalContext, + from_type: &DataType, + dest_type: DecimalDataType, +) -> Value { + if let DataType::Decimal(f) = from_type { + return decimal_to_decimal(arg, ctx, *f, dest_type); + } + + with_decimal_mapped_type!(|DECIMAL_TYPE| match dest_type { + DecimalDataType::DECIMAL_TYPE(size) => { + type T = DECIMAL_TYPE; + let result = match from_type { + DataType::Boolean => { + let arg = arg.try_downcast().unwrap(); + vectorize_1_arg::>(|a: bool, _| { + if a { + T::e(size.scale as u32) + } else { + T::zero() + } + })(arg, ctx) + } + + DataType::Number(ty) => { + if ty.is_float() { + match ty { + NumberDataType::Float32 => { + let arg = arg.try_downcast().unwrap(); + float_to_decimal::>(arg, ctx, size) + } + NumberDataType::Float64 => { + let arg = arg.try_downcast().unwrap(); + float_to_decimal::>(arg, ctx, size) + } + _ => unreachable!(), + } + } else { + with_integer_mapped_type!(|NUM_TYPE| match ty { + NumberDataType::NUM_TYPE => { + let arg = arg.try_downcast().unwrap(); + integer_to_decimal::>(arg, ctx, size) + } + _ => unreachable!(), + }) + } + } + DataType::String => { + let arg = arg.try_downcast().unwrap(); + string_to_decimal::(arg, ctx, size) + } + _ => unreachable!("to_decimal not support this DataType"), + }; + result.upcast_decimal(size) + } + }) +} + +fn convert_to_decimal_domain( + func_ctx: &FunctionContext, + domain: Domain, + dest_type: DecimalDataType, +) -> Option { + // Convert the domain to a Column. + // The first row is the min value, the second row is the max value. + let column = match domain { + Domain::Number(number_domain) => { + with_number_mapped_type!(|NUM_TYPE| match number_domain { + NumberDomain::NUM_TYPE(d) => { + let min = d.min; + let max = d.max; + NumberType::::from_data(vec![min, max]) + } + }) + } + Domain::Boolean(d) => { + let min = !d.has_false; + let max = d.has_true; + BooleanType::from_data(vec![min, max]) + } + Domain::Decimal(d) => { + with_decimal_mapped_type!(|DECIMAL| match d { + DecimalDomain::DECIMAL(d, size) => { + let min = d.min; + let max = d.max; + DecimalType::from_data_with_size(vec![min, max], size) + } + }) + } + Domain::String(d) => { + let min = d.min; + let max = d.max?; + StringType::from_data(vec![min, max]) + } + _ => { + return None; + } + }; + + let from_type = column.data_type(); + let value = Value::::Column(column); + let mut ctx = EvalContext { + generics: &[], + num_rows: 2, + func_ctx, + validity: None, + errors: None, + }; + let dest_size = dest_type.size(); + let res = convert_to_decimal(&value.as_ref(), &mut ctx, &from_type, dest_type); + + if ctx.errors.is_some() { + return None; + } + let decimal_col = res.as_column()?.as_decimal()?; + assert_eq!(decimal_col.len(), 2); + + Some(match decimal_col { + DecimalColumn::Decimal128(buf, size) => { + assert_eq!(&dest_size, size); + let (min, max) = unsafe { (*buf.get_unchecked(0), *buf.get_unchecked(1)) }; + DecimalDomain::Decimal128(SimpleDomain { min, max }, *size) + } + DecimalColumn::Decimal256(buf, size) => { + assert_eq!(&dest_size, size); + let (min, max) = unsafe { (*buf.get_unchecked(0), *buf.get_unchecked(1)) }; + DecimalDomain::Decimal256(SimpleDomain { min, max }, *size) + } + }) +} + +fn string_to_decimal( + from: ValueRef, + ctx: &mut EvalContext, + size: DecimalSize, +) -> Value> +where + T: Decimal + Mul, +{ + let f = |x: &[u8], builder: &mut Vec, ctx: &mut EvalContext| { + let value = match read_decimal_with_size::(x, size, true, ctx.func_ctx.rounding_mode) { + Ok((d, _)) => d, + Err(e) => { + ctx.set_error(builder.len(), e.message()); + T::zero() + } + }; + + builder.push(value); + }; + + vectorize_with_builder_1_arg::>(f)(from, ctx) +} + +fn integer_to_decimal( + from: ValueRef, + ctx: &mut EvalContext, + size: DecimalSize, +) -> Value> +where + T: Decimal + Mul, + for<'a> S::ScalarRef<'a>: Number + AsPrimitive, +{ + let multiplier = T::e(size.scale as u32); + + let min_for_precision = T::min_for_precision(size.precision); + let max_for_precision = T::max_for_precision(size.precision); + + let f = |x: S::ScalarRef<'_>, builder: &mut Vec, ctx: &mut EvalContext| { + let x = T::from_i128(x.as_()) * multiplier; + + if x > max_for_precision || x < min_for_precision { + ctx.set_error( + builder.len(), + concat!("Decimal overflow at line : ", line!()), + ); + builder.push(T::one()); + } else { + builder.push(x); + } + }; + + vectorize_with_builder_1_arg(f)(from, ctx) +} + +fn float_to_decimal( + from: ValueRef, + ctx: &mut EvalContext, + size: DecimalSize, +) -> Value> +where + for<'a> S::ScalarRef<'a>: Number + AsPrimitive, +{ + let multiplier: f64 = (10_f64).powi(size.scale as i32).as_(); + + let min_for_precision = T::min_for_precision(size.precision); + let max_for_precision = T::max_for_precision(size.precision); + + let f = |x: S::ScalarRef<'_>, builder: &mut Vec, ctx: &mut EvalContext| { + let mut x = x.as_() * multiplier; + if ctx.func_ctx.rounding_mode { + x = x.round(); + } + let x = T::from_float(x); + if x > max_for_precision || x < min_for_precision { + ctx.set_error( + builder.len(), + concat!("Decimal overflow at line : ", line!()), + ); + builder.push(T::one()); + } else { + builder.push(x); + } + }; + + vectorize_with_builder_1_arg(f)(from, ctx) +} + +#[inline] +fn get_round_val(x: T, scale: u32, ctx: &mut EvalContext) -> Option { + let mut round_val = None; + if ctx.func_ctx.rounding_mode && scale > 0 { + // Checking whether numbers need to be added or subtracted to calculate rounding + if let Some(r) = x.checked_rem(T::e(scale)) { + if let Some(m) = r.checked_div(T::e(scale - 1)) { + if m >= T::from_i128(5i64) { + round_val = Some(T::one()); + } else if m <= T::from_i128(-5i64) { + round_val = Some(T::minus_one()); + } + } + } + } + round_val +} + +fn decimal_256_to_128( + buffer: &ValueRef, + from_size: DecimalSize, + dest_size: DecimalSize, + ctx: &mut EvalContext, +) -> Value> { + let max = i128::max_for_precision(dest_size.precision); + let min = i128::min_for_precision(dest_size.precision); + + let buffer = buffer.try_downcast::>().unwrap(); + if dest_size.scale >= from_size.scale { + let factor = i256::e((dest_size.scale - from_size.scale) as u32); + + vectorize_with_builder_1_arg::, DecimalType>( + |x: i256, builder: &mut Vec, ctx: &mut EvalContext| match x.checked_mul(factor) { + Some(x) if x <= max && x >= min => builder.push(*x.low()), + _ => { + ctx.set_error( + builder.len(), + concat!("Decimal overflow at line : ", line!()), + ); + builder.push(i128::one()); + } + }, + )(buffer, ctx) + } else { + let scale_diff = (from_size.scale - dest_size.scale) as u32; + let factor = i256::e(scale_diff); + let source_factor = i256::e(from_size.scale as u32); + + vectorize_with_builder_1_arg::, DecimalType>( + |x: i256, builder: &mut Vec, ctx: &mut EvalContext| { + let round_val = get_round_val::(x, scale_diff, ctx); + let y = match (x.checked_div(factor), round_val) { + (Some(x), Some(round_val)) => x.checked_add(round_val), + (Some(x), None) => Some(x), + (None, _) => None, + }; + + match y { + Some(y) if (y <= max && y >= min) && (y != 0 || x / source_factor == 0) => { + builder.push(*y.low()); + } + _ => { + ctx.set_error( + builder.len(), + concat!("Decimal overflow at line : ", line!()), + ); + + builder.push(i128::one()); + } + } + }, + )(buffer, ctx) + } +} + +macro_rules! m_decimal_to_decimal { + ($from_size: expr, $dest_size: expr, $value: expr, $from_type_name: ty, $dest_type_name: ty, $ctx: expr) => { + type F = $from_type_name; + type T = $dest_type_name; + + let buffer: ValueRef> = $value.try_downcast().unwrap(); + // faster path + let result: Value> = if $from_size.scale == $dest_size.scale + && $from_size.precision <= $dest_size.precision + { + if F::MAX == T::MAX { + // 128 -> 128 or 256 -> 256 + return buffer.clone().to_owned().upcast_decimal($dest_size); + } else { + // 128 -> 256 + vectorize_1_arg::, DecimalType>(|x: F, _: &mut EvalContext| { + T::from(x) + })(buffer, $ctx) + } + } else if $from_size.scale > $dest_size.scale { + let scale_diff = ($from_size.scale - $dest_size.scale) as u32; + let factor = T::e(scale_diff); + let max = T::max_for_precision($dest_size.precision); + let min = T::min_for_precision($dest_size.precision); + + let source_factor = F::e($from_size.scale as u32); + + vectorize_with_builder_1_arg::, DecimalType>( + |x: F, builder: &mut Vec, ctx: &mut EvalContext| { + let x = T::from(x); + let round_val = get_round_val::(x, scale_diff, ctx); + let y = match (x.checked_div(factor), round_val) { + (Some(x), Some(round_val)) => x.checked_add(round_val), + (Some(x), None) => Some(x), + (None, _) => None, + }; + + match y { + Some(y) if y <= max && y >= min && (y != 0 || x / source_factor == 0) => { + builder.push(y as T); + } + _ => { + ctx.set_error( + builder.len(), + concat!("Decimal overflow at line : ", line!()), + ); + builder.push(T::one()); + } + } + }, + )(buffer, $ctx) + } else { + let factor = T::e(($dest_size.scale - $from_size.scale) as u32); + let min = T::min_for_precision($dest_size.precision); + let max = T::max_for_precision($dest_size.precision); + + vectorize_with_builder_1_arg::, DecimalType>( + |x: F, builder: &mut Vec, ctx: &mut EvalContext| { + let x = T::from(x); + match x.checked_mul(factor) { + Some(x) if x <= max && x >= min => { + builder.push(x as T); + } + _ => { + ctx.set_error( + builder.len(), + concat!("Decimal overflow at line : ", line!()), + ); + builder.push(T::one()); + } + } + }, + )(buffer, $ctx) + }; + + result.upcast_decimal($dest_size) + }; +} + +fn decimal_to_decimal( + arg: &ValueRef, + ctx: &mut EvalContext, + from_type: DecimalDataType, + dest_type: DecimalDataType, +) -> Value { + let from_size = from_type.size(); + let dest_size = dest_type.size(); + match (from_type, dest_type) { + (DecimalDataType::Decimal128(_), DecimalDataType::Decimal128(_)) => { + m_decimal_to_decimal! {from_size, dest_size, arg, i128, i128, ctx} + } + (DecimalDataType::Decimal128(_), DecimalDataType::Decimal256(_)) => { + m_decimal_to_decimal! {from_size, dest_size, arg, i128, i256, ctx} + } + (DecimalDataType::Decimal256(_), DecimalDataType::Decimal128(_)) => { + let value = decimal_256_to_128(arg, from_size, dest_size, ctx); + value.upcast_decimal(dest_size) + } + (DecimalDataType::Decimal256(_), DecimalDataType::Decimal256(_)) => { + m_decimal_to_decimal! {from_size, dest_size, arg, i256, i256, ctx} + } + } +} + +trait DecimalConvert { + fn convert(t: T, _scale: i32) -> U; +} + +impl DecimalConvert for F32 { + fn convert(t: i128, scale: i32) -> F32 { + let div = 10f32.powi(scale); + ((t as f32) / div).into() + } +} + +impl DecimalConvert for F64 { + fn convert(t: i128, scale: i32) -> F64 { + let div = 10f64.powi(scale); + ((t as f64) / div).into() + } +} + +impl DecimalConvert for F32 { + fn convert(t: i256, scale: i32) -> F32 { + let div = 10f32.powi(scale); + (f32::from(t) / div).into() + } +} + +impl DecimalConvert for F64 { + fn convert(t: i256, scale: i32) -> F64 { + let div = 10f64.powi(scale); + (f64::from(t) / div).into() + } +} + +fn decimal_to_float( + arg: &ValueRef, + from_type: DataType, + ctx: &mut EvalContext, +) -> Value +where + T: Number, + T: DecimalConvert, + T: DecimalConvert, +{ + let from_type = from_type.as_decimal().unwrap(); + + let result = with_decimal_mapped_type!(|DECIMAL_TYPE| match from_type { + DecimalDataType::DECIMAL_TYPE(from_size) => { + let value = arg.try_downcast().unwrap(); + let scale = from_size.scale as i32; + vectorize_1_arg::, NumberType>( + |x, _ctx: &mut EvalContext| T::convert(x, scale), + )(value, ctx) + } + }); + + result.upcast() +} + +fn decimal_to_int( + arg: &ValueRef, + from_type: DataType, + ctx: &mut EvalContext, +) -> Value { + let from_type = from_type.as_decimal().unwrap(); + + let result = with_decimal_mapped_type!(|DECIMAL_TYPE| match from_type { + DecimalDataType::DECIMAL_TYPE(from_size) => { + let value = arg.try_downcast().unwrap(); + vectorize_with_builder_1_arg::, NumberType>( + |x, builder: &mut Vec, ctx: &mut EvalContext| match x + .to_int(from_size.scale, ctx.func_ctx.rounding_mode) + { + Some(x) => builder.push(x), + None => { + ctx.set_error(builder.len(), "decimal cast to int overflow"); + builder.push(T::default()) + } + }, + )(value, ctx) + } + }); + + result.upcast() +} diff --git a/src/query/functions/src/scalars/decimal/comparison.rs b/src/query/functions/src/scalars/decimal/comparison.rs new file mode 100644 index 000000000000..3c49b6630dab --- /dev/null +++ b/src/query/functions/src/scalars/decimal/comparison.rs @@ -0,0 +1,130 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::cmp::Ord; +use std::ops::*; +use std::sync::Arc; + +use databend_common_expression::type_check::common_super_type; +use databend_common_expression::types::decimal::*; +use databend_common_expression::types::*; +use databend_common_expression::vectorize_2_arg; +use databend_common_expression::Domain; +use databend_common_expression::EvalContext; +use databend_common_expression::Function; +use databend_common_expression::FunctionEval; +use databend_common_expression::FunctionRegistry; +use databend_common_expression::FunctionSignature; +use databend_common_expression::SimpleDomainCmp; +use databend_common_expression::Value; +use databend_common_expression::ValueRef; +use ethnum::i256; + +macro_rules! register_decimal_compare_op { + ($registry: expr, $name: expr, $op: ident, $domain_op: tt) => { + $registry.register_function_factory($name, |_, args_type| { + if args_type.len() != 2 { + return None; + } + + let has_nullable = args_type.iter().any(|x| x.is_nullable_or_null()); + let args_type: Vec = args_type.iter().map(|x| x.remove_nullable()).collect(); + + // Only works for one of is decimal types + if !args_type[0].is_decimal() && !args_type[1].is_decimal() { + return None; + } + + let common_type = common_super_type(args_type[0].clone(), args_type[1].clone(), &[])?; + + if !common_type.is_decimal() { + return None; + } + + // Comparison between different decimal types must be same siganature types + let function = Function { + signature: FunctionSignature { + name: $name.to_string(), + args_type: vec![common_type.clone(), common_type.clone()], + return_type: DataType::Boolean, + }, + eval: FunctionEval::Scalar { + calc_domain: Box::new(|_, d| { + let new_domain = match (&d[0], &d[1]) { + ( + Domain::Decimal(DecimalDomain::Decimal128(d1, _)), + Domain::Decimal(DecimalDomain::Decimal128(d2, _)), + ) => d1.$domain_op(d2), + ( + Domain::Decimal(DecimalDomain::Decimal256(d1, _)), + Domain::Decimal(DecimalDomain::Decimal256(d2, _)), + ) => d1.$domain_op(d2), + _ => unreachable!("Expect two same decimal domains, got {:?}", d), + }; + new_domain.map(|d| Domain::Boolean(d)) + }), + eval: Box::new(move |args, ctx| { + op_decimal! { &args[0], &args[1], common_type, $op, ctx} + }), + }, + }; + if has_nullable { + Some(Arc::new(function.passthrough_nullable())) + } else { + Some(Arc::new(function)) + } + }); + }; +} + +macro_rules! op_decimal { + ($a: expr, $b: expr, $common_type: expr, $op: ident, $ctx: expr) => { + match $common_type { + DataType::Decimal(DecimalDataType::Decimal128(_)) => { + let f = |a: i128, b: i128, _: &mut EvalContext| -> bool { a.cmp(&b).$op() }; + compare_decimal($a, $b, f, $ctx) + } + DataType::Decimal(DecimalDataType::Decimal256(_)) => { + let f = |a: i256, b: i256, _: &mut EvalContext| -> bool { a.cmp(&b).$op() }; + compare_decimal($a, $b, f, $ctx) + } + _ => unreachable!(), + } + }; +} + +fn compare_decimal( + a: &ValueRef, + b: &ValueRef, + f: F, + ctx: &mut EvalContext, +) -> Value +where + T: Decimal, + F: Fn(T, T, &mut EvalContext) -> bool + Copy + Send + Sync, +{ + let a = a.try_downcast().unwrap(); + let b = b.try_downcast().unwrap(); + let value = vectorize_2_arg::, DecimalType, BooleanType>(f)(a, b, ctx); + value.upcast() +} + +pub fn register_decimal_compare_op(registry: &mut FunctionRegistry) { + register_decimal_compare_op!(registry, "lt", is_lt, domain_lt); + register_decimal_compare_op!(registry, "eq", is_eq, domain_eq); + register_decimal_compare_op!(registry, "gt", is_gt, domain_gt); + register_decimal_compare_op!(registry, "lte", is_le, domain_lte); + register_decimal_compare_op!(registry, "gte", is_ge, domain_gte); + register_decimal_compare_op!(registry, "noteq", is_ne, domain_noteq); +} diff --git a/src/query/functions/src/scalars/decimal/math.rs b/src/query/functions/src/scalars/decimal/math.rs new file mode 100644 index 000000000000..7ed5976b858b --- /dev/null +++ b/src/query/functions/src/scalars/decimal/math.rs @@ -0,0 +1,289 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::cmp::Ord; +use std::ops::*; +use std::sync::Arc; + +use databend_common_expression::types::decimal::*; +use databend_common_expression::types::*; +use databend_common_expression::vectorize_1_arg; +use databend_common_expression::with_decimal_mapped_type; +use databend_common_expression::EvalContext; +use databend_common_expression::Function; +use databend_common_expression::FunctionDomain; +use databend_common_expression::FunctionEval; +use databend_common_expression::FunctionRegistry; +use databend_common_expression::FunctionSignature; +use databend_common_expression::Value; +use databend_common_expression::ValueRef; +use ethnum::i256; + +pub fn register_decimal_math(registry: &mut FunctionRegistry) { + let factory = |params: &[usize], args_type: &[DataType], round_mode: RoundMode| { + if args_type.is_empty() { + return None; + } + + let from_type = args_type[0].remove_nullable(); + if !matches!(from_type, DataType::Decimal(_)) { + return None; + } + + let from_decimal_type = from_type.as_decimal().unwrap(); + + let scale = if params.is_empty() { + 0 + } else { + params[0] as i64 - 76 + }; + + let decimal_size = DecimalSize { + precision: from_decimal_type.precision(), + scale: scale.clamp(0, from_decimal_type.scale() as i64) as u8, + }; + + let dest_decimal_type = DecimalDataType::from_size(decimal_size).ok()?; + let name = format!("{:?}", round_mode).to_lowercase(); + + let mut sig_args_type = args_type.to_owned(); + sig_args_type[0] = from_type.clone(); + let f = Function { + signature: FunctionSignature { + name, + args_type: sig_args_type, + return_type: DataType::Decimal(dest_decimal_type), + }, + eval: FunctionEval::Scalar { + calc_domain: Box::new(move |_ctx, _d| FunctionDomain::Full), + eval: Box::new(move |args, ctx| { + decimal_math( + &args[0], + ctx, + from_type.clone(), + dest_decimal_type, + scale, + round_mode, + ) + }), + }, + }; + + if args_type[0].is_nullable() { + Some(f.passthrough_nullable()) + } else { + Some(f) + } + }; + + for m in [ + RoundMode::Round, + RoundMode::Truncate, + RoundMode::Ceil, + RoundMode::Floor, + ] { + let name = format!("{:?}", m).to_lowercase(); + registry.register_function_factory(&name, move |params, args_type| { + Some(Arc::new(factory(params, args_type, m)?)) + }); + } +} + +#[derive(Copy, Clone, Debug)] +enum RoundMode { + Round, + Truncate, + Floor, + Ceil, +} + +fn decimal_round_positive( + value: ValueRef>, + source_scale: i64, + target_scale: i64, + ctx: &mut EvalContext, +) -> Value> +where + T: Decimal + From + DivAssign + Div + Add + Sub, +{ + let power_of_ten = T::e((source_scale - target_scale) as u32); + let addition = power_of_ten / T::from(2); + vectorize_1_arg::, DecimalType>(|a, _| { + if a < T::zero() { + (a - addition) / power_of_ten + } else { + (a + addition) / power_of_ten + } + })(value, ctx) +} + +fn decimal_round_negative( + value: ValueRef>, + source_scale: i64, + target_scale: i64, + ctx: &mut EvalContext, +) -> Value> +where + T: Decimal + + From + + DivAssign + + Div + + Add + + Sub + + Mul, +{ + let divide_power_of_ten = T::e((source_scale - target_scale) as u32); + let addition = divide_power_of_ten / T::from(2); + let multiply_power_of_ten = T::e((-target_scale) as u32); + + vectorize_1_arg::, DecimalType>(|a, _| { + let a = if a < T::zero() { + a - addition + } else { + a + addition + }; + a / divide_power_of_ten * multiply_power_of_ten + })(value, ctx) +} + +// if round mode is ceil, truncate should add one value +fn decimal_truncate_positive( + value: ValueRef>, + source_scale: i64, + target_scale: i64, + ctx: &mut EvalContext, +) -> Value> +where + T: Decimal + From + DivAssign + Div + Add + Sub, +{ + let power_of_ten = T::e((source_scale - target_scale) as u32); + vectorize_1_arg::, DecimalType>(|a, _| a / power_of_ten)(value, ctx) +} + +fn decimal_truncate_negative( + value: ValueRef>, + source_scale: i64, + target_scale: i64, + ctx: &mut EvalContext, +) -> Value> +where + T: Decimal + + From + + DivAssign + + Div + + Add + + Sub + + Mul, +{ + let divide_power_of_ten = T::e((source_scale - target_scale) as u32); + let multiply_power_of_ten = T::e((-target_scale) as u32); + + vectorize_1_arg::, DecimalType>(|a, _| { + a / divide_power_of_ten * multiply_power_of_ten + })(value, ctx) +} + +fn decimal_floor( + value: ValueRef>, + source_scale: i64, + ctx: &mut EvalContext, +) -> Value> +where + T: Decimal + + From + + DivAssign + + Div + + Add + + Sub + + Mul, +{ + let power_of_ten = T::e(source_scale as u32); + + vectorize_1_arg::, DecimalType>(|a, _| { + if a < T::zero() { + // below 0 we ceil the number (e.g. -10.5 -> -11) + ((a + T::one()) / power_of_ten) - T::one() + } else { + a / power_of_ten + } + })(value, ctx) +} + +fn decimal_ceil( + value: ValueRef>, + source_scale: i64, + ctx: &mut EvalContext, +) -> Value> +where + T: Decimal + + From + + DivAssign + + Div + + Add + + Sub + + Mul, +{ + let power_of_ten = T::e(source_scale as u32); + + vectorize_1_arg::, DecimalType>(|a, _| { + if a <= T::zero() { + a / power_of_ten + } else { + ((a - T::one()) / power_of_ten) + T::one() + } + })(value, ctx) +} + +fn decimal_math( + arg: &ValueRef, + ctx: &mut EvalContext, + from_type: DataType, + dest_type: DecimalDataType, + target_scale: i64, + mode: RoundMode, +) -> Value { + let from_decimal_type = from_type.as_decimal().unwrap(); + let source_scale = from_decimal_type.scale() as i64; + + if source_scale < target_scale { + return arg.clone().to_owned(); + } + + let none_negative = target_scale >= 0; + + with_decimal_mapped_type!(|DECIMAL_TYPE| match from_decimal_type { + DecimalDataType::DECIMAL_TYPE(_) => { + let value = arg.try_downcast::>().unwrap(); + + let result = match (none_negative, mode) { + (true, RoundMode::Round) => { + decimal_round_positive::<_>(value, source_scale, target_scale, ctx) + } + (true, RoundMode::Truncate) => { + decimal_truncate_positive::<_>(value, source_scale, target_scale, ctx) + } + (false, RoundMode::Round) => { + decimal_round_negative::<_>(value, source_scale, target_scale, ctx) + } + (false, RoundMode::Truncate) => { + decimal_truncate_negative::<_>(value, source_scale, target_scale, ctx) + } + (_, RoundMode::Floor) => decimal_floor::<_>(value, source_scale, ctx), + (_, RoundMode::Ceil) => decimal_ceil::<_>(value, source_scale, ctx), + }; + + result.upcast_decimal(dest_type.size()) + } + }) +} diff --git a/src/query/functions/src/scalars/decimal/mod.rs b/src/query/functions/src/scalars/decimal/mod.rs new file mode 100644 index 000000000000..12407ca97278 --- /dev/null +++ b/src/query/functions/src/scalars/decimal/mod.rs @@ -0,0 +1,25 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod arithmetic; +mod cast; +mod comparison; +mod math; + +pub(crate) use arithmetic::register_decimal_arithmetic; +pub(crate) use cast::register_decimal_to_float; +pub(crate) use cast::register_decimal_to_int; +pub(crate) use cast::register_to_decimal; +pub(crate) use comparison::register_decimal_compare_op; +pub(crate) use math::register_decimal_math; diff --git a/src/query/functions/src/scalars/mod.rs b/src/query/functions/src/scalars/mod.rs index 0e482e58f513..464ee93a5aa0 100644 --- a/src/query/functions/src/scalars/mod.rs +++ b/src/query/functions/src/scalars/mod.rs @@ -57,7 +57,7 @@ pub fn register(registry: &mut FunctionRegistry) { geo_h3::register(registry); hash::register(registry); other::register(registry); - decimal::register(registry); + decimal::register_to_decimal(registry); vector::register(registry); bitmap::register(registry); } diff --git a/src/query/functions/tests/it/scalars/cast.rs b/src/query/functions/tests/it/scalars/cast.rs index ec360d1cf77e..9ed9a6a79a5a 100644 --- a/src/query/functions/tests/it/scalars/cast.rs +++ b/src/query/functions/tests/it/scalars/cast.rs @@ -665,6 +665,11 @@ fn test_cast_between_string_and_decimal(file: &mut impl Write, is_try: bool) { format!("{prefix}CAST('-1.0e+10' AS DECIMAL(11, 0))"), &[], ); + run_ast( + file, + format!("{prefix}CAST('-0.000000' AS DECIMAL(11, 0))"), + &[], + ); } fn gen_bitmap_data() -> Column { diff --git a/src/query/functions/tests/it/scalars/comparison.rs b/src/query/functions/tests/it/scalars/comparison.rs index 7a3f8a7c4564..c93cc6046858 100644 --- a/src/query/functions/tests/it/scalars/comparison.rs +++ b/src/query/functions/tests/it/scalars/comparison.rs @@ -40,6 +40,7 @@ fn test_eq(file: &mut impl Write) { run_ast(file, "null=null", &[]); run_ast(file, "1=2", &[]); run_ast(file, "1.0=1", &[]); + run_ast(file, "2.222>2.11", &[]); run_ast(file, "true=null", &[]); run_ast(file, "true=false", &[]); run_ast(file, "false=false", &[]); diff --git a/src/query/functions/tests/it/scalars/testdata/cast.txt b/src/query/functions/tests/it/scalars/testdata/cast.txt index 084da87a5bf5..cf09cc4fe637 100644 --- a/src/query/functions/tests/it/scalars/testdata/cast.txt +++ b/src/query/functions/tests/it/scalars/testdata/cast.txt @@ -1080,6 +1080,15 @@ output domain : {-10000000000..=-10000000000} output : -10000000000 +ast : CAST('-0.000000' AS DECIMAL(11, 0)) +raw expr : CAST('-0.000000' AS Decimal(11, 0)) +checked expr : to_decimal(11, 0)("-0.000000") +optimized expr : 0_d128(11,0) +output type : Decimal(11, 0) +output domain : {0..=0} +output : 0 + + ast : CAST(0 AS BOOLEAN) raw expr : CAST(0 AS Boolean) checked expr : to_boolean(0_u8) @@ -3070,6 +3079,15 @@ output domain : {-10000000000..=-10000000000} output : -10000000000 +ast : TRY_CAST('-0.000000' AS DECIMAL(11, 0)) +raw expr : TRY_CAST('-0.000000' AS Decimal(11, 0)) +checked expr : try_to_decimal(11, 0)("-0.000000") +optimized expr : 0_d128(11,0) +output type : Decimal(11, 0) NULL +output domain : {0..=0} +output : 0 + + ast : TRY_CAST(0 AS BOOLEAN) raw expr : TRY_CAST(0 AS Boolean) checked expr : try_to_boolean(0_u8) diff --git a/src/query/functions/tests/it/scalars/testdata/comparison.txt b/src/query/functions/tests/it/scalars/testdata/comparison.txt index 866182f55989..97194c73aef2 100644 --- a/src/query/functions/tests/it/scalars/testdata/comparison.txt +++ b/src/query/functions/tests/it/scalars/testdata/comparison.txt @@ -34,6 +34,15 @@ output domain : {TRUE} output : true +ast : 2.222>2.11 +raw expr : gt(2.222, 2.11) +checked expr : gt(2.222_d128(4,3), to_decimal(4, 3)(2.11_d128(3,2))) +optimized expr : true +output type : Boolean +output domain : {TRUE} +output : true + + ast : true=null raw expr : eq(true, NULL) checked expr : eq(CAST(true AS Boolean NULL), CAST(NULL AS Boolean NULL)) diff --git a/src/query/storages/parquet/src/parquet_rs/statistics/column.rs b/src/query/storages/parquet/src/parquet_rs/statistics/column.rs index da2f92fca7b5..416c12022c4e 100644 --- a/src/query/storages/parquet/src/parquet_rs/statistics/column.rs +++ b/src/query/storages/parquet/src/parquet_rs/statistics/column.rs @@ -58,14 +58,8 @@ pub fn convert_column_statistics(s: &Statistics, typ: &TableDataType) -> ColumnS Scalar::Decimal(DecimalScalar::Decimal128(i128::from(min), *size)), ), TableDataType::Decimal(DecimalDataType::Decimal256(size)) => ( - Scalar::Decimal(DecimalScalar::Decimal256( - I256::from_i64(max as i64), - *size, - )), - Scalar::Decimal(DecimalScalar::Decimal256( - I256::from_i64(min as i64), - *size, - )), + Scalar::Decimal(DecimalScalar::Decimal256(I256::from_i128(max), *size)), + Scalar::Decimal(DecimalScalar::Decimal256(I256::from_i128(min), *size)), ), _ => (Scalar::Null, Scalar::Null), } @@ -95,8 +89,8 @@ pub fn convert_column_statistics(s: &Statistics, typ: &TableDataType) -> ColumnS Scalar::Decimal(DecimalScalar::Decimal128(i128::from(min), *size)), ), TableDataType::Decimal(DecimalDataType::Decimal256(size)) => ( - Scalar::Decimal(DecimalScalar::Decimal256(I256::from_i64(max), *size)), - Scalar::Decimal(DecimalScalar::Decimal256(I256::from_i64(min), *size)), + Scalar::Decimal(DecimalScalar::Decimal256(I256::from_i128(max), *size)), + Scalar::Decimal(DecimalScalar::Decimal256(I256::from_i128(min), *size)), ), _ => (Scalar::Null, Scalar::Null), } diff --git a/src/query/storages/parquet/src/parquet_rs/statistics/page.rs b/src/query/storages/parquet/src/parquet_rs/statistics/page.rs index 27186c6ab0f0..d4e12246af2e 100644 --- a/src/query/storages/parquet/src/parquet_rs/statistics/page.rs +++ b/src/query/storages/parquet/src/parquet_rs/statistics/page.rs @@ -151,8 +151,8 @@ fn convert_page_index_int32( Scalar::Decimal(DecimalScalar::Decimal128(i128::from(min), *size)), ), TableDataType::Decimal(DecimalDataType::Decimal256(size)) => ( - Scalar::Decimal(DecimalScalar::Decimal256(I256::from_i64(max as i64), *size)), - Scalar::Decimal(DecimalScalar::Decimal256(I256::from_i64(min as i64), *size)), + Scalar::Decimal(DecimalScalar::Decimal256(I256::from_i128(max), *size)), + Scalar::Decimal(DecimalScalar::Decimal256(I256::from_i128(min), *size)), ), _ => unreachable!(), }; @@ -187,8 +187,8 @@ fn convert_page_index_int64( Scalar::Decimal(DecimalScalar::Decimal128(i128::from(min), *size)), ), TableDataType::Decimal(DecimalDataType::Decimal256(size)) => ( - Scalar::Decimal(DecimalScalar::Decimal256(I256::from_i64(max), *size)), - Scalar::Decimal(DecimalScalar::Decimal256(I256::from_i64(min), *size)), + Scalar::Decimal(DecimalScalar::Decimal256(I256::from_i128(max), *size)), + Scalar::Decimal(DecimalScalar::Decimal256(I256::from_i128(min), *size)), ), _ => unreachable!(), }; From 9361cf3781bfd80a1133eba05893536960750e93 Mon Sep 17 00:00:00 2001 From: Winter Zhang Date: Tue, 19 Dec 2023 09:33:01 +0800 Subject: [PATCH 14/20] chore(executor): add exchange row and bytes profile (#14067) --- .../pipeline/core/src/processors/processor.rs | 4 ++++ .../pipeline/core/src/processors/profile.rs | 16 +++++++++++++++- src/query/pipeline/sinks/src/async_sink.rs | 7 +++++++ .../src/processors/transforms/transform.rs | 7 +++++++ .../src/api/rpc/exchange/exchange_sink_writer.rs | 15 ++++++++++++++- .../rpc/exchange/serde/exchange_serializer.rs | 14 ++++++++++++++ .../tests/it/storages/testdata/columns_table.txt | 2 ++ .../system/src/processor_profile_table.rs | 14 ++++++++++++++ 8 files changed, 77 insertions(+), 2 deletions(-) diff --git a/src/query/pipeline/core/src/processors/processor.rs b/src/query/pipeline/core/src/processors/processor.rs index 13248fb4efa0..e9e01d0af9c4 100644 --- a/src/query/pipeline/core/src/processors/processor.rs +++ b/src/query/pipeline/core/src/processors/processor.rs @@ -246,4 +246,8 @@ impl Processor for Box { fn details_status(&self) -> Option { (**self).details_status() } + + fn record_profile(&self, profile: &Profile) { + (**self).record_profile(profile) + } } diff --git a/src/query/pipeline/core/src/processors/profile.rs b/src/query/pipeline/core/src/processors/profile.rs index d5275bb238d4..c68b447c2977 100644 --- a/src/query/pipeline/core/src/processors/profile.rs +++ b/src/query/pipeline/core/src/processors/profile.rs @@ -17,7 +17,7 @@ use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; use std::sync::Arc; -#[derive(Default)] +#[derive(Default, Debug)] pub struct Profile { /// The id of processor pub pid: usize, @@ -33,6 +33,9 @@ pub struct Profile { /// The time spent to wait in nanoseconds, usually used to /// measure the time spent on waiting for I/O pub wait_time: AtomicU64, + + pub exchange_rows: AtomicUsize, + pub exchange_bytes: AtomicUsize, } impl Profile { @@ -42,6 +45,8 @@ impl Profile { p_name, cpu_time: AtomicU64::new(0), wait_time: AtomicU64::new(0), + exchange_rows: AtomicUsize::new(0), + exchange_bytes: AtomicUsize::new(0), plan_id: scope.as_ref().map(|x| x.id), plan_name: scope.as_ref().map(|x| x.name.clone()), plan_parent_id: scope.as_ref().and_then(|x| x.parent_id), @@ -60,6 +65,9 @@ pub struct PlanProfile { /// The time spent to wait in nanoseconds, usually used to /// measure the time spent on waiting for I/O pub wait_time: usize, + + pub exchange_rows: usize, + pub exchange_bytes: usize, } impl PlanProfile { @@ -70,17 +78,23 @@ impl PlanProfile { parent_id: profile.plan_parent_id, cpu_time: profile.cpu_time.load(Ordering::SeqCst) as usize, wait_time: profile.wait_time.load(Ordering::SeqCst) as usize, + exchange_rows: profile.exchange_rows.load(Ordering::SeqCst), + exchange_bytes: profile.exchange_bytes.load(Ordering::SeqCst), } } pub fn accumulate(&mut self, profile: &Profile) { self.cpu_time += profile.cpu_time.load(Ordering::SeqCst) as usize; self.wait_time += profile.wait_time.load(Ordering::SeqCst) as usize; + self.exchange_rows += profile.exchange_rows.load(Ordering::SeqCst); + self.exchange_bytes += profile.exchange_bytes.load(Ordering::SeqCst); } pub fn merge(&mut self, profile: &PlanProfile) { self.cpu_time += profile.cpu_time; self.wait_time += profile.wait_time; + self.exchange_rows += profile.exchange_rows; + self.exchange_bytes += profile.exchange_bytes; } } diff --git a/src/query/pipeline/sinks/src/async_sink.rs b/src/query/pipeline/sinks/src/async_sink.rs index 47c170cb9ffb..be7ed120acea 100644 --- a/src/query/pipeline/sinks/src/async_sink.rs +++ b/src/query/pipeline/sinks/src/async_sink.rs @@ -22,6 +22,7 @@ use databend_common_base::runtime::TrySpawn; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; use databend_common_expression::DataBlock; +use databend_common_pipeline_core::processors::profile::Profile; use databend_common_pipeline_core::processors::Event; use databend_common_pipeline_core::processors::InputPort; use databend_common_pipeline_core::processors::Processor; @@ -46,6 +47,8 @@ pub trait AsyncSink: Send { fn details_status(&self) -> Option { None } + + fn record_profile(&self, _profile: &Profile) {} } pub struct AsyncSinker { @@ -170,4 +173,8 @@ impl Processor for AsyncSinker { fn details_status(&self) -> Option { self.inner.as_ref().and_then(|x| x.details_status()) } + + fn record_profile(&self, profile: &Profile) { + self.inner.as_ref().unwrap().record_profile(profile); + } } diff --git a/src/query/pipeline/transforms/src/processors/transforms/transform.rs b/src/query/pipeline/transforms/src/processors/transforms/transform.rs index ae6ec7a0b785..91c8cde21e00 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/transform.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/transform.rs @@ -21,6 +21,7 @@ use databend_common_exception::Result; use databend_common_expression::BlockMetaInfo; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::DataBlock; +use databend_common_pipeline_core::processors::profile::Profile; use databend_common_pipeline_core::processors::Event; use databend_common_pipeline_core::processors::InputPort; use databend_common_pipeline_core::processors::OutputPort; @@ -44,6 +45,8 @@ pub trait Transform: Send { fn on_finish(&mut self) -> Result<()> { Ok(()) } + + fn record_profile(&self, _: &Profile) {} } pub struct Transformer { @@ -122,6 +125,10 @@ impl Processor for Transformer { Ok(()) } + + fn record_profile(&self, profile: &Profile) { + self.transform.record_profile(profile) + } } impl Transformer { diff --git a/src/query/service/src/api/rpc/exchange/exchange_sink_writer.rs b/src/query/service/src/api/rpc/exchange/exchange_sink_writer.rs index e076a5e9d8ec..4de8c9941cc6 100644 --- a/src/query/service/src/api/rpc/exchange/exchange_sink_writer.rs +++ b/src/query/service/src/api/rpc/exchange/exchange_sink_writer.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; use std::sync::Arc; use databend_common_catalog::table_context::TableContext; @@ -20,6 +22,7 @@ use databend_common_exception::Result; use databend_common_expression::BlockMetaInfoDowncast; use databend_common_expression::DataBlock; use databend_common_metrics::transform::*; +use databend_common_pipeline_core::processors::profile::Profile; use databend_common_pipeline_core::processors::InputPort; use databend_common_pipeline_core::processors::Processor; use databend_common_pipeline_core::processors::ProcessorPtr; @@ -38,6 +41,7 @@ pub struct ExchangeWriterSink { source: String, destination: String, fragment: usize, + exchange_bytes: AtomicUsize, } impl ExchangeWriterSink { @@ -54,6 +58,7 @@ impl ExchangeWriterSink { source: source_id.to_string(), destination: destination_id.to_string(), fragment: fragment_id, + exchange_bytes: AtomicUsize::new(0), }) } } @@ -96,11 +101,19 @@ impl AsyncSink for ExchangeWriterSink { { metrics_inc_exchange_write_count(count); metrics_inc_exchange_write_bytes(bytes); + self.exchange_bytes.fetch_add(bytes, Ordering::Relaxed); } Ok(false) } + fn record_profile(&self, profile: &Profile) { + profile.exchange_bytes.fetch_add( + self.exchange_bytes.swap(0, Ordering::Relaxed), + Ordering::Relaxed, + ); + } + fn details_status(&self) -> Option { #[derive(Debug)] #[allow(dead_code)] @@ -113,7 +126,7 @@ impl AsyncSink for ExchangeWriterSink { Some(format!("{:?}", Display { source: self.source.clone(), destination: self.destination.clone(), - fragment: self.fragment + fragment: self.fragment, })) } } diff --git a/src/query/service/src/api/rpc/exchange/serde/exchange_serializer.rs b/src/query/service/src/api/rpc/exchange/serde/exchange_serializer.rs index ffc021f536da..70ce171cd935 100644 --- a/src/query/service/src/api/rpc/exchange/serde/exchange_serializer.rs +++ b/src/query/service/src/api/rpc/exchange/serde/exchange_serializer.rs @@ -14,6 +14,8 @@ use std::fmt::Debug; use std::fmt::Formatter; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; use std::sync::Arc; use databend_common_arrow::arrow::chunk::Chunk; @@ -29,6 +31,7 @@ use databend_common_expression::BlockMetaInfoPtr; use databend_common_expression::DataBlock; use databend_common_io::prelude::bincode_serialize_into_buf; use databend_common_io::prelude::BinaryWrite; +use databend_common_pipeline_core::processors::profile::Profile; use databend_common_pipeline_core::processors::InputPort; use databend_common_pipeline_core::processors::OutputPort; use databend_common_pipeline_core::processors::ProcessorPtr; @@ -95,6 +98,7 @@ impl BlockMetaInfo for ExchangeSerializeMeta { pub struct TransformExchangeSerializer { options: WriteOptions, ipc_fields: Vec, + exchange_rows: AtomicUsize, } impl TransformExchangeSerializer { @@ -120,6 +124,7 @@ impl TransformExchangeSerializer { TransformExchangeSerializer { ipc_fields, options: WriteOptions { compression }, + exchange_rows: AtomicUsize::new(0), }, ))) } @@ -129,8 +134,17 @@ impl Transform for TransformExchangeSerializer { const NAME: &'static str = "ExchangeSerializerTransform"; fn transform(&mut self, data_block: DataBlock) -> Result { + self.exchange_rows + .fetch_add(data_block.num_rows(), Ordering::Relaxed); serialize_block(0, data_block, &self.ipc_fields, &self.options) } + + fn record_profile(&self, profile: &Profile) { + profile.exchange_rows.fetch_add( + self.exchange_rows.swap(0, Ordering::Relaxed), + Ordering::Relaxed, + ); + } } pub struct TransformScatterExchangeSerializer { diff --git a/src/query/service/tests/it/storages/testdata/columns_table.txt b/src/query/service/tests/it/storages/testdata/columns_table.txt index deb5372e781e..8a8db09227cf 100644 --- a/src/query/service/tests/it/storages/testdata/columns_table.txt +++ b/src/query/service/tests/it/storages/testdata/columns_table.txt @@ -132,6 +132,8 @@ DB.Table: 'system'.'columns', Table: columns-table_id:1, ver:0, Engine: SystemCo | 'exception_code' | 'system' | 'task_history' | 'Int64' | 'BIGINT' | '' | '' | 'NO' | '' | | 'exception_text' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | | 'exception_text' | 'system' | 'task_history' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | +| 'exchange_bytes' | 'system' | 'processor_profile' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | +| 'exchange_rows' | 'system' | 'processor_profile' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | | 'execution_info' | 'system' | 'query_profile' | 'Variant' | 'VARIANT' | '' | '' | 'NO' | '' | | 'extra' | 'information_schema' | 'columns' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | | 'extra' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | diff --git a/src/query/storages/system/src/processor_profile_table.rs b/src/query/storages/system/src/processor_profile_table.rs index fb4a9576cd9f..b551bcae7a88 100644 --- a/src/query/storages/system/src/processor_profile_table.rs +++ b/src/query/storages/system/src/processor_profile_table.rs @@ -62,6 +62,8 @@ impl SyncSystemTable for ProcessorProfileTable { let mut plan_name: Vec>> = Vec::with_capacity(total_size); let mut cpu_time: Vec = Vec::with_capacity(total_size); let mut wait_time: Vec = Vec::with_capacity(total_size); + let mut exchange_rows: Vec = Vec::with_capacity(total_size); + let mut exchange_bytes: Vec = Vec::with_capacity(total_size); for (query_id, query_profiles) in queries_profiles { for query_profile in query_profiles { @@ -75,6 +77,8 @@ impl SyncSystemTable for ProcessorProfileTable { cpu_time.push(query_profile.cpu_time.load(Ordering::Relaxed)); wait_time.push(query_profile.wait_time.load(Ordering::Relaxed)); + exchange_rows.push(query_profile.exchange_rows.load(Ordering::Relaxed) as u64); + exchange_bytes.push(query_profile.exchange_bytes.load(Ordering::Relaxed) as u64); } } @@ -88,6 +92,8 @@ impl SyncSystemTable for ProcessorProfileTable { StringType::from_opt_data(plan_name), UInt64Type::from_data(cpu_time), UInt64Type::from_data(wait_time), + UInt64Type::from_data(exchange_rows), + UInt64Type::from_data(exchange_bytes), ])) } } @@ -113,6 +119,14 @@ impl ProcessorProfileTable { ), TableField::new("cpu_time", TableDataType::Number(NumberDataType::UInt64)), TableField::new("wait_time", TableDataType::Number(NumberDataType::UInt64)), + TableField::new( + "exchange_rows", + TableDataType::Number(NumberDataType::UInt64), + ), + TableField::new( + "exchange_bytes", + TableDataType::Number(NumberDataType::UInt64), + ), ]); let table_info = TableInfo { From 39f2becddf94874c4678a6a31f9153d71395a236 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Tue, 19 Dec 2023 10:12:22 +0800 Subject: [PATCH 15/20] fix(query): add missing server_version for query log (#14074) --- src/query/service/src/interpreters/common/query_log.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/query/service/src/interpreters/common/query_log.rs b/src/query/service/src/interpreters/common/query_log.rs index 150f920f8066..057f58988e88 100644 --- a/src/query/service/src/interpreters/common/query_log.rs +++ b/src/query/service/src/interpreters/common/query_log.rs @@ -17,6 +17,7 @@ use std::sync::Arc; use std::time::SystemTime; use databend_common_config::GlobalConfig; +use databend_common_config::DATABEND_COMMIT_VERSION; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_storages_system::LogType; @@ -182,7 +183,7 @@ impl InterpreterQueryLog { exception_code, exception_text, stack_trace, - server_version: "".to_string(), + server_version: DATABEND_COMMIT_VERSION.to_string(), session_settings, extra: "".to_string(), has_profiles: false, @@ -319,7 +320,7 @@ impl InterpreterQueryLog { exception_code, exception_text, stack_trace, - server_version: "".to_string(), + server_version: DATABEND_COMMIT_VERSION.to_string(), session_settings, extra: "".to_string(), has_profiles, From f2cfe7d98a0fa277007cca87178031a83bed0dc8 Mon Sep 17 00:00:00 2001 From: Winter Zhang Date: Tue, 19 Dec 2023 10:13:47 +0800 Subject: [PATCH 16/20] fix(query): fix panic when drop UncompressedBuffer (#14068) fix(query): fix panic when drop uncompression buffer --- .../fuse/src/io/read/block/decompressor.rs | 57 +++++++++++++------ 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/src/query/storages/fuse/src/io/read/block/decompressor.rs b/src/query/storages/fuse/src/io/read/block/decompressor.rs index 1c12e5ec9bc1..968daaedfc30 100644 --- a/src/query/storages/fuse/src/io/read/block/decompressor.rs +++ b/src/query/storages/fuse/src/io/read/block/decompressor.rs @@ -44,22 +44,26 @@ impl UncompressedBuffer { }) } - pub fn clear(&self) { - if self.used.fetch_add(1, Ordering::SeqCst) != 0 { - self.used.fetch_sub(1, Ordering::SeqCst); + pub fn clear(self: &Arc) { + let guard = self.borrow_mut(); + + if !guard.is_unique_borrow_mut() { panic!( "UncompressedBuffer cannot be accessed between multiple threads at the same time." ); } drop(std::mem::take(self.buffer_mut())); - self.used.fetch_sub(1, Ordering::SeqCst); } #[allow(clippy::mut_from_ref)] pub(in crate::io::read::block::decompressor) fn buffer_mut(&self) -> &mut Vec { unsafe { &mut *self.buffer.get() } } + + pub(in crate::io::read::block::decompressor) fn borrow_mut(self: &Arc) -> UsedGuard { + UsedGuard::create(self) + } } pub struct BuffedBasicDecompressor>> { @@ -90,8 +94,9 @@ where I: Iterator> fn advance(&mut self) -> Result<(), Error> { if let Some(page) = self.current.as_mut() { if self.was_decompressed { - if self.uncompressed_buffer.used.fetch_add(1, Ordering::SeqCst) != 0 { - self.uncompressed_buffer.used.fetch_sub(1, Ordering::SeqCst); + let guard = self.uncompressed_buffer.borrow_mut(); + + if !guard.is_unique_borrow_mut() { return Err(Error::FeatureNotSupported(String::from( "UncompressedBuffer cannot be accessed between multiple threads at the same time.", ))); @@ -104,16 +109,15 @@ where I: Iterator> *borrow_buffer = std::mem::take(page.buffer_mut()); } } - - self.uncompressed_buffer.used.fetch_sub(1, Ordering::SeqCst); } } self.current = match self.iter.next() { None => None, Some(page) => { - if self.uncompressed_buffer.used.fetch_add(1, Ordering::SeqCst) != 0 { - self.uncompressed_buffer.used.fetch_sub(1, Ordering::SeqCst); + let guard = self.uncompressed_buffer.borrow_mut(); + + if !guard.is_unique_borrow_mut() { return Err(Error::FeatureNotSupported(String::from( "UncompressedBuffer cannot be accessed between multiple threads at the same time.", ))); @@ -126,8 +130,6 @@ where I: Iterator> decompress(page, self.uncompressed_buffer.buffer_mut())? }; - self.uncompressed_buffer.used.fetch_sub(1, Ordering::SeqCst); - Some(decompress_page) } }; @@ -149,10 +151,9 @@ where I: Iterator> impl>> Drop for BuffedBasicDecompressor { fn drop(&mut self) { if let Some(page) = self.current.as_mut() { - if !std::thread::panicking() - && self.uncompressed_buffer.used.fetch_add(1, Ordering::SeqCst) != 0 - { - self.uncompressed_buffer.used.fetch_sub(1, Ordering::SeqCst); + let guard = self.uncompressed_buffer.borrow_mut(); + + if !std::thread::panicking() && !guard.is_unique_borrow_mut() { panic!( "UncompressedBuffer cannot be accessed between multiple threads at the same time." ); @@ -165,8 +166,30 @@ impl>> Drop for BuffedBasicDeco *borrow_buffer = std::mem::take(page.buffer_mut()); } } + } + } +} - self.uncompressed_buffer.used.fetch_sub(1, Ordering::SeqCst); +struct UsedGuard { + unique_mut: bool, + inner: Arc, +} + +impl UsedGuard { + pub fn create(inner: &Arc) -> UsedGuard { + let used = inner.used.fetch_add(1, Ordering::SeqCst); + UsedGuard { + unique_mut: used == 0, + inner: inner.clone(), } } + pub fn is_unique_borrow_mut(&self) -> bool { + self.unique_mut + } +} + +impl Drop for UsedGuard { + fn drop(&mut self) { + self.inner.used.fetch_sub(1, Ordering::SeqCst); + } } From 52637445fb6b021952fe5917ce3de8bf3d02ff61 Mon Sep 17 00:00:00 2001 From: baishen Date: Tue, 19 Dec 2023 16:03:58 +0800 Subject: [PATCH 17/20] feat(query): Support show virtual columns (#14038) * feat(query): Support show virtual columns * fix tests * fix tests * fix --------- Co-authored-by: BohuTANG --- Cargo.lock | 1 + src/meta/api/src/schema_api_impl.rs | 39 +++-- src/meta/api/src/schema_api_test_suite.rs | 5 + src/meta/app/src/schema/virtual_column.rs | 3 + src/query/ast/src/ast/format/ast_format.rs | 26 ++++ src/query/ast/src/ast/statements/statement.rs | 2 + .../ast/src/ast/statements/virtual_column.rs | 52 ++++++- src/query/ast/src/parser/statement.rs | 36 ++++- src/query/ast/src/visitors/visitor.rs | 2 + src/query/ast/src/visitors/visitor_mut.rs | 2 + src/query/ast/src/visitors/walk.rs | 1 + src/query/ast/src/visitors/walk_mut.rs | 1 + .../ast/tests/it/testdata/statement-error.txt | 2 +- src/query/ast/tests/it/testdata/statement.txt | 3 + .../src/databases/system/system_database.rs | 2 + .../interpreter_virtual_column_alter.rs | 1 + .../interpreter_virtual_column_create.rs | 1 + .../interpreter_virtual_column_drop.rs | 1 + .../it/storages/testdata/columns_table.txt | 5 + src/query/sql/src/planner/binder/binder.rs | 1 + .../src/planner/binder/ddl/virtual_column.rs | 75 ++++++++++ .../src/planner/plans/ddl/virtual_column.rs | 3 + src/query/sql/src/planner/plans/plan.rs | 1 + src/query/storages/system/Cargo.toml | 1 + .../storages/system/src/columns_table.rs | 141 ++++++++++-------- src/query/storages/system/src/lib.rs | 2 + .../system/src/virtual_columns_table.rs | 138 +++++++++++++++++ .../01_ee_system/01_0002_virtual_column.test | 10 ++ ..._0002_ddl_create_drop_virtual_columns.test | 9 ++ 29 files changed, 481 insertions(+), 85 deletions(-) create mode 100644 src/query/storages/system/src/virtual_columns_table.rs diff --git a/Cargo.lock b/Cargo.lock index e5ac5940a8cd..729925cc2f3f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3816,6 +3816,7 @@ dependencies = [ "databend-common-functions", "databend-common-meta-api", "databend-common-meta-app", + "databend-common-meta-types", "databend-common-metrics", "databend-common-pipeline-core", "databend-common-pipeline-sources", diff --git a/src/meta/api/src/schema_api_impl.rs b/src/meta/api/src/schema_api_impl.rs index 93ed01d4ef2a..f378f4902cd4 100644 --- a/src/meta/api/src/schema_api_impl.rs +++ b/src/meta/api/src/schema_api_impl.rs @@ -1402,15 +1402,19 @@ impl + ?Sized> SchemaApi for KV { get_pb_value(self, &req.name_ident).await?; if old_virtual_column_opt.is_some() { - return Err(KVAppError::AppError(AppError::VirtualColumnAlreadyExists( - VirtualColumnAlreadyExists::new( - req.name_ident.table_id, - format!( - "create virtual column with tenant: {} table_id: {}", - req.name_ident.tenant, req.name_ident.table_id + if req.if_not_exists { + return Ok(CreateVirtualColumnReply {}); + } else { + return Err(KVAppError::AppError(AppError::VirtualColumnAlreadyExists( + VirtualColumnAlreadyExists::new( + req.name_ident.table_id, + format!( + "create virtual column with tenant: {} table_id: {}", + req.name_ident.tenant, req.name_ident.table_id + ), ), - ), - ))); + ))); + } } let virtual_column_meta = VirtualColumnMeta { table_id: req.name_ident.table_id, @@ -1464,7 +1468,16 @@ impl + ?Sized> SchemaApi for KV { trials.next().unwrap()?; let (seq, old_virtual_column_meta) = - get_virtual_column_by_id_or_err(self, &req.name_ident, ctx).await?; + match get_virtual_column_by_id_or_err(self, &req.name_ident, ctx).await { + Ok((seq, old_virtual_column_meta)) => (seq, old_virtual_column_meta), + Err(err) => { + if req.if_exists { + return Ok(UpdateVirtualColumnReply {}); + } else { + return Err(err); + } + } + }; let virtual_column_meta = VirtualColumnMeta { table_id: req.name_ident.table_id, @@ -1517,7 +1530,13 @@ impl + ?Sized> SchemaApi for KV { loop { trials.next().unwrap()?; - let (_, _) = get_virtual_column_by_id_or_err(self, &req.name_ident, ctx).await?; + if let Err(err) = get_virtual_column_by_id_or_err(self, &req.name_ident, ctx).await { + if req.if_exists { + return Ok(DropVirtualColumnReply {}); + } else { + return Err(err); + } + } // Drop virtual column by deleting this record: // (tenant, table_id) -> virtual_column_meta diff --git a/src/meta/api/src/schema_api_test_suite.rs b/src/meta/api/src/schema_api_test_suite.rs index 5e11cb977b1b..9c87b1a755a2 100644 --- a/src/meta/api/src/schema_api_test_suite.rs +++ b/src/meta/api/src/schema_api_test_suite.rs @@ -5768,6 +5768,7 @@ impl SchemaApiTestSuite { { info!("--- create virtual column"); let req = CreateVirtualColumnReq { + if_not_exists: false, name_ident: name_ident.clone(), virtual_columns: vec!["variant:k1".to_string(), "variant[1]".to_string()], }; @@ -5776,6 +5777,7 @@ impl SchemaApiTestSuite { info!("--- create virtual column again"); let req = CreateVirtualColumnReq { + if_not_exists: false, name_ident: name_ident.clone(), virtual_columns: vec!["variant:k1".to_string(), "variant[1]".to_string()], }; @@ -5810,6 +5812,7 @@ impl SchemaApiTestSuite { { info!("--- update virtual column"); let req = UpdateVirtualColumnReq { + if_exists: false, name_ident: name_ident.clone(), virtual_columns: vec!["variant:k2".to_string(), "variant[2]".to_string()], }; @@ -5835,6 +5838,7 @@ impl SchemaApiTestSuite { { info!("--- drop virtual column"); let req = DropVirtualColumnReq { + if_exists: false, name_ident: name_ident.clone(), }; @@ -5855,6 +5859,7 @@ impl SchemaApiTestSuite { { info!("--- update virtual column after drop"); let req = UpdateVirtualColumnReq { + if_exists: false, name_ident: name_ident.clone(), virtual_columns: vec!["variant:k3".to_string(), "variant[3]".to_string()], }; diff --git a/src/meta/app/src/schema/virtual_column.rs b/src/meta/app/src/schema/virtual_column.rs index 8af8894a7b9e..f4dda52d65f1 100644 --- a/src/meta/app/src/schema/virtual_column.rs +++ b/src/meta/app/src/schema/virtual_column.rs @@ -56,6 +56,7 @@ pub struct VirtualColumnMeta { #[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq)] pub struct CreateVirtualColumnReq { + pub if_not_exists: bool, pub name_ident: VirtualColumnNameIdent, pub virtual_columns: Vec, } @@ -75,6 +76,7 @@ pub struct CreateVirtualColumnReply {} #[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq)] pub struct UpdateVirtualColumnReq { + pub if_exists: bool, pub name_ident: VirtualColumnNameIdent, pub virtual_columns: Vec, } @@ -94,6 +96,7 @@ pub struct UpdateVirtualColumnReply {} #[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq)] pub struct DropVirtualColumnReq { + pub if_exists: bool, pub name_ident: VirtualColumnNameIdent, } diff --git a/src/query/ast/src/ast/format/ast_format.rs b/src/query/ast/src/ast/format/ast_format.rs index 3ab8b1c42266..dc471b28839f 100644 --- a/src/query/ast/src/ast/format/ast_format.rs +++ b/src/query/ast/src/ast/format/ast_format.rs @@ -1839,6 +1839,32 @@ impl<'ast> Visitor<'ast> for AstFormatVisitor { self.children.push(node); } + fn visit_show_virtual_columns(&mut self, stmt: &'ast ShowVirtualColumnsStmt) { + let mut children = Vec::new(); + if let Some(database) = &stmt.database { + let database_name = format!("Database {}", database); + let database_format_ctx = AstFormatContext::new(database_name); + let database_node = FormatTreeNode::new(database_format_ctx); + children.push(database_node); + } + + if let Some(table) = &stmt.database { + let table_name = format!("Table {}", table); + let table_format_ctx = AstFormatContext::new(table_name); + let table_node = FormatTreeNode::new(table_format_ctx); + children.push(table_node); + } + + if let Some(limit) = &stmt.limit { + self.visit_show_limit(limit); + children.push(self.children.pop().unwrap()); + } + let name = "ShowVirtualColumns".to_string(); + let format_ctx = AstFormatContext::with_children(name, children.len()); + let node = FormatTreeNode::with_children(format_ctx, children); + self.children.push(node); + } + fn visit_show_users(&mut self) { let name = "ShowUsers".to_string(); let format_ctx = AstFormatContext::new(name); diff --git a/src/query/ast/src/ast/statements/statement.rs b/src/query/ast/src/ast/statements/statement.rs index d6ca54d4fb4c..ac3e1b3cf5e1 100644 --- a/src/query/ast/src/ast/statements/statement.rs +++ b/src/query/ast/src/ast/statements/statement.rs @@ -157,6 +157,7 @@ pub enum Statement { AlterVirtualColumn(AlterVirtualColumnStmt), DropVirtualColumn(DropVirtualColumnStmt), RefreshVirtualColumn(RefreshVirtualColumnStmt), + ShowVirtualColumns(ShowVirtualColumnsStmt), // User ShowUsers, @@ -469,6 +470,7 @@ impl Display for Statement { Statement::AlterVirtualColumn(stmt) => write!(f, "{stmt}")?, Statement::DropVirtualColumn(stmt) => write!(f, "{stmt}")?, Statement::RefreshVirtualColumn(stmt) => write!(f, "{stmt}")?, + Statement::ShowVirtualColumns(stmt) => write!(f, "{stmt}")?, Statement::ShowUsers => write!(f, "SHOW USERS")?, Statement::ShowRoles => write!(f, "SHOW ROLES")?, Statement::CreateUser(stmt) => write!(f, "{stmt}")?, diff --git a/src/query/ast/src/ast/statements/virtual_column.rs b/src/query/ast/src/ast/statements/virtual_column.rs index 239bb2a9c78a..65f22e969672 100644 --- a/src/query/ast/src/ast/statements/virtual_column.rs +++ b/src/query/ast/src/ast/statements/virtual_column.rs @@ -19,9 +19,11 @@ use crate::ast::write_comma_separated_list; use crate::ast::write_dot_separated_list; use crate::ast::Expr; use crate::ast::Identifier; +use crate::ast::ShowLimit; #[derive(Debug, Clone, PartialEq)] pub struct CreateVirtualColumnStmt { + pub if_not_exists: bool, pub catalog: Option, pub database: Option, pub table: Identifier, @@ -31,7 +33,11 @@ pub struct CreateVirtualColumnStmt { impl Display for CreateVirtualColumnStmt { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "CREATE VIRTUAL COLUMN (")?; + write!(f, "CREATE VIRTUAL COLUMN ")?; + if self.if_not_exists { + write!(f, "IF NOT EXISTS ")?; + } + write!(f, "(")?; write_comma_separated_list(f, &self.virtual_columns)?; write!(f, ") FOR ")?; write_dot_separated_list( @@ -47,6 +53,7 @@ impl Display for CreateVirtualColumnStmt { #[derive(Debug, Clone, PartialEq)] pub struct AlterVirtualColumnStmt { + pub if_exists: bool, pub catalog: Option, pub database: Option, pub table: Identifier, @@ -56,7 +63,11 @@ pub struct AlterVirtualColumnStmt { impl Display for AlterVirtualColumnStmt { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "ALTER VIRTUAL COLUMN (")?; + write!(f, "ALTER VIRTUAL COLUMN ")?; + if self.if_exists { + write!(f, "IF EXISTS ")?; + } + write!(f, "(")?; write_comma_separated_list(f, &self.virtual_columns)?; write!(f, ") FOR ")?; write_dot_separated_list( @@ -72,6 +83,7 @@ impl Display for AlterVirtualColumnStmt { #[derive(Debug, Clone, PartialEq)] pub struct DropVirtualColumnStmt { + pub if_exists: bool, pub catalog: Option, pub database: Option, pub table: Identifier, @@ -79,7 +91,11 @@ pub struct DropVirtualColumnStmt { impl Display for DropVirtualColumnStmt { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "DROP VIRTUAL COLUMN FOR ")?; + write!(f, "DROP VIRTUAL COLUMN ")?; + if self.if_exists { + write!(f, "IF EXISTS ")?; + } + write!(f, "FOR ")?; write_dot_separated_list( f, self.catalog @@ -112,3 +128,33 @@ impl Display for RefreshVirtualColumnStmt { Ok(()) } } + +#[derive(Debug, Clone, PartialEq)] +pub struct ShowVirtualColumnsStmt { + pub catalog: Option, + pub database: Option, + pub table: Option, + pub limit: Option, +} + +impl Display for ShowVirtualColumnsStmt { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + write!(f, "SHOW VIRTUAL COLUMNS")?; + if let Some(table) = &self.table { + write!(f, "FROM {}", table)?; + } + if let Some(database) = &self.database { + write!(f, " FROM ")?; + if let Some(catalog) = &self.catalog { + write!(f, "{catalog}.",)?; + } + write!(f, "{database}")?; + } + + if let Some(limit) = &self.limit { + write!(f, " {limit}")?; + } + + Ok(()) + } +} diff --git a/src/query/ast/src/parser/statement.rs b/src/query/ast/src/parser/statement.rs index 5dd2d967b4bc..745219128de9 100644 --- a/src/query/ast/src/parser/statement.rs +++ b/src/query/ast/src/parser/statement.rs @@ -958,10 +958,11 @@ pub fn statement(i: Input) -> IResult { let create_virtual_column = map( rule! { - CREATE ~ VIRTUAL ~ COLUMN ~ ^"(" ~ ^#comma_separated_list1(expr) ~ ^")" ~ FOR ~ #dot_separated_idents_1_to_3 + CREATE ~ VIRTUAL ~ COLUMN ~ ( IF ~ ^NOT ~ ^EXISTS )? ~ ^"(" ~ ^#comma_separated_list1(expr) ~ ^")" ~ FOR ~ #dot_separated_idents_1_to_3 }, - |(_, _, _, _, virtual_columns, _, _, (catalog, database, table))| { + |(_, _, _, opt_if_not_exists, _, virtual_columns, _, _, (catalog, database, table))| { Statement::CreateVirtualColumn(CreateVirtualColumnStmt { + if_not_exists: opt_if_not_exists.is_some(), catalog, database, table, @@ -972,10 +973,11 @@ pub fn statement(i: Input) -> IResult { let alter_virtual_column = map( rule! { - ALTER ~ VIRTUAL ~ COLUMN ~ ^"(" ~ ^#comma_separated_list1(expr) ~ ^")" ~ FOR ~ #dot_separated_idents_1_to_3 + ALTER ~ VIRTUAL ~ COLUMN ~ ( IF ~ ^EXISTS )? ~ ^"(" ~ ^#comma_separated_list1(expr) ~ ^")" ~ FOR ~ #dot_separated_idents_1_to_3 }, - |(_, _, _, _, virtual_columns, _, _, (catalog, database, table))| { + |(_, _, _, opt_if_exists, _, virtual_columns, _, _, (catalog, database, table))| { Statement::AlterVirtualColumn(AlterVirtualColumnStmt { + if_exists: opt_if_exists.is_some(), catalog, database, table, @@ -986,10 +988,11 @@ pub fn statement(i: Input) -> IResult { let drop_virtual_column = map( rule! { - DROP ~ VIRTUAL ~ COLUMN ~ FOR ~ #dot_separated_idents_1_to_3 + DROP ~ VIRTUAL ~ COLUMN ~ ( IF ~ ^EXISTS )? ~ FOR ~ #dot_separated_idents_1_to_3 }, - |(_, _, _, _, (catalog, database, table))| { + |(_, _, _, opt_if_exists, _, (catalog, database, table))| { Statement::DropVirtualColumn(DropVirtualColumnStmt { + if_exists: opt_if_exists.is_some(), catalog, database, table, @@ -1010,6 +1013,26 @@ pub fn statement(i: Input) -> IResult { }, ); + let show_virtual_columns = map( + rule! { + SHOW ~ VIRTUAL ~ COLUMNS ~ (( FROM | IN ) ~ #ident)? ~ (( FROM | IN ) ~ ^#dot_separated_idents_1_to_2)? ~ #show_limit? + }, + |(_, _, _, opt_table, opt_db, limit)| { + let table = opt_table.map(|(_, table)| table); + let (catalog, database) = match opt_db { + Some((_, (Some(c), d))) => (Some(c), Some(d)), + Some((_, (None, d))) => (None, Some(d)), + _ => (None, None), + }; + Statement::ShowVirtualColumns(ShowVirtualColumnsStmt { + catalog, + database, + table, + limit, + }) + }, + ); + let show_users = value(Statement::ShowUsers, rule! { SHOW ~ USERS }); let create_user = map( rule! { @@ -1739,6 +1762,7 @@ pub fn statement(i: Input) -> IResult { | #alter_virtual_column: "`ALTER VIRTUAL COLUMN (expr, ...) FOR [.]`" | #drop_virtual_column: "`DROP VIRTUAL COLUMN FOR [.]
`" | #refresh_virtual_column: "`REFRESH VIRTUAL COLUMN FOR [.]
`" + | #show_virtual_columns : "`SHOW VIRTUAL COLUMNS FROM
[FROM|IN .] []`" ), rule!( #show_users : "`SHOW USERS`" diff --git a/src/query/ast/src/visitors/visitor.rs b/src/query/ast/src/visitors/visitor.rs index d3263e61dbbb..c264243f8113 100644 --- a/src/query/ast/src/visitors/visitor.rs +++ b/src/query/ast/src/visitors/visitor.rs @@ -578,6 +578,8 @@ pub trait Visitor<'ast>: Sized { fn visit_refresh_virtual_column(&mut self, _stmt: &'ast RefreshVirtualColumnStmt) {} + fn visit_show_virtual_columns(&mut self, _stmt: &'ast ShowVirtualColumnsStmt) {} + fn visit_show_users(&mut self) {} fn visit_create_user(&mut self, _stmt: &'ast CreateUserStmt) {} diff --git a/src/query/ast/src/visitors/visitor_mut.rs b/src/query/ast/src/visitors/visitor_mut.rs index f14c8513ed3c..fa1cf82bc237 100644 --- a/src/query/ast/src/visitors/visitor_mut.rs +++ b/src/query/ast/src/visitors/visitor_mut.rs @@ -592,6 +592,8 @@ pub trait VisitorMut: Sized { fn visit_refresh_virtual_column(&mut self, _stmt: &mut RefreshVirtualColumnStmt) {} + fn visit_show_virtual_columns(&mut self, _stmt: &mut ShowVirtualColumnsStmt) {} + fn visit_show_users(&mut self) {} fn visit_create_user(&mut self, _stmt: &mut CreateUserStmt) {} diff --git a/src/query/ast/src/visitors/walk.rs b/src/query/ast/src/visitors/walk.rs index 319f5e35dff0..13813a2d3ea7 100644 --- a/src/query/ast/src/visitors/walk.rs +++ b/src/query/ast/src/visitors/walk.rs @@ -436,6 +436,7 @@ pub fn walk_statement<'a, V: Visitor<'a>>(visitor: &mut V, statement: &'a Statem Statement::AlterVirtualColumn(stmt) => visitor.visit_alter_virtual_column(stmt), Statement::DropVirtualColumn(stmt) => visitor.visit_drop_virtual_column(stmt), Statement::RefreshVirtualColumn(stmt) => visitor.visit_refresh_virtual_column(stmt), + Statement::ShowVirtualColumns(stmt) => visitor.visit_show_virtual_columns(stmt), Statement::ShowUsers => visitor.visit_show_users(), Statement::ShowRoles => visitor.visit_show_roles(), Statement::CreateUser(stmt) => visitor.visit_create_user(stmt), diff --git a/src/query/ast/src/visitors/walk_mut.rs b/src/query/ast/src/visitors/walk_mut.rs index 97928f849ade..990da8deebb5 100644 --- a/src/query/ast/src/visitors/walk_mut.rs +++ b/src/query/ast/src/visitors/walk_mut.rs @@ -441,6 +441,7 @@ pub fn walk_statement_mut(visitor: &mut V, statement: &mut Statem Statement::AlterVirtualColumn(stmt) => visitor.visit_alter_virtual_column(stmt), Statement::DropVirtualColumn(stmt) => visitor.visit_drop_virtual_column(stmt), Statement::RefreshVirtualColumn(stmt) => visitor.visit_refresh_virtual_column(stmt), + Statement::ShowVirtualColumns(stmt) => visitor.visit_show_virtual_columns(stmt), Statement::ShowUsers => visitor.visit_show_users(), Statement::ShowRoles => visitor.visit_show_roles(), Statement::CreateUser(stmt) => visitor.visit_create_user(stmt), diff --git a/src/query/ast/tests/it/testdata/statement-error.txt b/src/query/ast/tests/it/testdata/statement-error.txt index 2d1a6f527bfa..5236c275c3c9 100644 --- a/src/query/ast/tests/it/testdata/statement-error.txt +++ b/src/query/ast/tests/it/testdata/statement-error.txt @@ -237,7 +237,7 @@ error: --> SQL:1:6 | 1 | SHOW GRANT FOR ROLE 'role1'; - | ^^^^^ unexpected `GRANT`, expecting `GRANTS`, `CREATE`, `NETWORK`, `STREAMS`, `CATALOGS`, `FUNCTIONS`, `DATABASES`, `CONNECTIONS`, `TABLE_FUNCTIONS`, `DROP`, `TABLE`, `ROLES`, `SHARE`, `TASKS`, `INDEXES`, `COLUMNS`, `PROCESSLIST`, `STAGES`, `TABLES`, `SHARES`, `ENGINES`, `METRICS`, `SETTINGS`, `LOCKS`, `SCHEMAS`, `FIELDS`, `USERS`, `FILE`, or `FULL` + | ^^^^^ unexpected `GRANT`, expecting `GRANTS`, `CREATE`, `NETWORK`, `VIRTUAL`, `STREAMS`, `CATALOGS`, `FUNCTIONS`, `DATABASES`, `CONNECTIONS`, `TABLE_FUNCTIONS`, `DROP`, `TABLE`, `ROLES`, `SHARE`, `TASKS`, `INDEXES`, `COLUMNS`, `PROCESSLIST`, `STAGES`, `TABLES`, `SHARES`, `ENGINES`, `METRICS`, `SETTINGS`, `LOCKS`, `SCHEMAS`, `FIELDS`, `USERS`, `FILE`, or `FULL` ---------- Input ---------- diff --git a/src/query/ast/tests/it/testdata/statement.txt b/src/query/ast/tests/it/testdata/statement.txt index b35c29f9dc78..0804b72ca7b9 100644 --- a/src/query/ast/tests/it/testdata/statement.txt +++ b/src/query/ast/tests/it/testdata/statement.txt @@ -13511,6 +13511,7 @@ CREATE VIRTUAL COLUMN (a['k1']['k2'], b[0][1]) FOR t ---------- AST ------------ CreateVirtualColumn( CreateVirtualColumnStmt { + if_not_exists: false, catalog: None, database: None, table: Identifier { @@ -13625,6 +13626,7 @@ ALTER VIRTUAL COLUMN (a['k1']['k2'], b[0][1]) FOR t ---------- AST ------------ AlterVirtualColumn( AlterVirtualColumnStmt { + if_exists: false, catalog: None, database: None, table: Identifier { @@ -13739,6 +13741,7 @@ DROP VIRTUAL COLUMN FOR t ---------- AST ------------ DropVirtualColumn( DropVirtualColumnStmt { + if_exists: false, catalog: None, database: None, table: Identifier { diff --git a/src/query/service/src/databases/system/system_database.rs b/src/query/service/src/databases/system/system_database.rs index 591d4976389f..9fd11b1e7c45 100644 --- a/src/query/service/src/databases/system/system_database.rs +++ b/src/query/service/src/databases/system/system_database.rs @@ -59,6 +59,7 @@ use databend_common_storages_system::TasksTable; use databend_common_storages_system::TempFilesTable; use databend_common_storages_system::TracingTable; use databend_common_storages_system::UsersTable; +use databend_common_storages_system::VirtualColumnsTable; use crate::catalogs::InMemoryMetas; use crate::databases::Database; @@ -125,6 +126,7 @@ impl SystemDatabase { TaskHistoryTable::create(sys_db_meta.next_table_id()), ProcessorProfileTable::create(sys_db_meta.next_table_id()), LocksTable::create(sys_db_meta.next_table_id()), + VirtualColumnsTable::create(sys_db_meta.next_table_id()), ]; let disable_tables = Self::disable_system_tables(); diff --git a/src/query/service/src/interpreters/interpreter_virtual_column_alter.rs b/src/query/service/src/interpreters/interpreter_virtual_column_alter.rs index 41f6dd7711c4..e6cf3234eee3 100644 --- a/src/query/service/src/interpreters/interpreter_virtual_column_alter.rs +++ b/src/query/service/src/interpreters/interpreter_virtual_column_alter.rs @@ -68,6 +68,7 @@ impl Interpreter for AlterVirtualColumnInterpreter { let catalog = self.ctx.get_catalog(&catalog_name).await?; let update_virtual_column_req = UpdateVirtualColumnReq { + if_exists: self.plan.if_exists, name_ident: VirtualColumnNameIdent { tenant, table_id }, virtual_columns: self.plan.virtual_columns.clone(), }; diff --git a/src/query/service/src/interpreters/interpreter_virtual_column_create.rs b/src/query/service/src/interpreters/interpreter_virtual_column_create.rs index 019806353d64..ac832154833d 100644 --- a/src/query/service/src/interpreters/interpreter_virtual_column_create.rs +++ b/src/query/service/src/interpreters/interpreter_virtual_column_create.rs @@ -68,6 +68,7 @@ impl Interpreter for CreateVirtualColumnInterpreter { let catalog = self.ctx.get_catalog(&catalog_name).await?; let create_virtual_column_req = CreateVirtualColumnReq { + if_not_exists: self.plan.if_not_exists, name_ident: VirtualColumnNameIdent { tenant, table_id }, virtual_columns: self.plan.virtual_columns.clone(), }; diff --git a/src/query/service/src/interpreters/interpreter_virtual_column_drop.rs b/src/query/service/src/interpreters/interpreter_virtual_column_drop.rs index cce1366be6da..4817b9b7a766 100644 --- a/src/query/service/src/interpreters/interpreter_virtual_column_drop.rs +++ b/src/query/service/src/interpreters/interpreter_virtual_column_drop.rs @@ -68,6 +68,7 @@ impl Interpreter for DropVirtualColumnInterpreter { let catalog = self.ctx.get_catalog(&catalog_name).await?; let drop_virtual_column_req = DropVirtualColumnReq { + if_exists: self.plan.if_exists, name_ident: VirtualColumnNameIdent { tenant, table_id }, }; diff --git a/src/query/service/tests/it/storages/testdata/columns_table.txt b/src/query/service/tests/it/storages/testdata/columns_table.txt index 8a8db09227cf..f4c61bf33535 100644 --- a/src/query/service/tests/it/storages/testdata/columns_table.txt +++ b/src/query/service/tests/it/storages/testdata/columns_table.txt @@ -71,6 +71,7 @@ DB.Table: 'system'.'columns', Table: columns-table_id:1, ver:0, Engine: SystemCo | 'created_on' | 'system' | 'tables' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | | 'created_on' | 'system' | 'tables_with_history' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | | 'created_on' | 'system' | 'tasks' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'created_on' | 'system' | 'virtual_columns' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | | 'creator' | 'system' | 'background_jobs' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | | 'creator' | 'system' | 'background_tasks' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | | 'creator' | 'system' | 'stages' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | @@ -91,6 +92,7 @@ DB.Table: 'system'.'columns', Table: columns-table_id:1, ver:0, Engine: SystemCo | 'database' | 'system' | 'streams' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | | 'database' | 'system' | 'tables' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | | 'database' | 'system' | 'tables_with_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'database' | 'system' | 'virtual_columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | | 'database_id' | 'system' | 'background_tasks' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | | 'database_id' | 'system' | 'databases' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | | 'databases' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | @@ -336,6 +338,7 @@ DB.Table: 'system'.'columns', Table: columns-table_id:1, ver:0, Engine: SystemCo | 'syntax' | 'system' | 'functions' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | | 'table' | 'system' | 'clustering_history' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | | 'table' | 'system' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | +| 'table' | 'system' | 'virtual_columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | | 'table_catalog' | 'information_schema' | 'columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | | 'table_catalog' | 'information_schema' | 'key_column_usage' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | | 'table_catalog' | 'information_schema' | 'statistics' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | @@ -381,6 +384,7 @@ DB.Table: 'system'.'columns', Table: columns-table_id:1, ver:0, Engine: SystemCo | 'updated_on' | 'system' | 'streams' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | | 'updated_on' | 'system' | 'tables' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | | 'updated_on' | 'system' | 'tables_with_history' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'updated_on' | 'system' | 'virtual_columns' | 'Nullable(Timestamp)' | 'TIMESTAMP' | '' | '' | 'YES' | '' | | 'user' | 'system' | 'locks' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | | 'user' | 'system' | 'processes' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | | 'user_agent' | 'system' | 'query_log' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | @@ -392,6 +396,7 @@ DB.Table: 'system'.'columns', Table: columns-table_id:1, ver:0, Engine: SystemCo | 'version' | 'system' | 'clusters' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | | 'version' | 'system' | 'credits' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | | 'view_definition' | 'information_schema' | 'views' | 'NULL' | 'NULL' | '' | '' | 'NO' | '' | +| 'virtual_columns' | 'system' | 'virtual_columns' | 'String' | 'VARCHAR' | '' | '' | 'NO' | '' | | 'wait_time' | 'system' | 'processor_profile' | 'UInt64' | 'BIGINT UNSIGNED' | '' | '' | 'NO' | '' | | 'warehouse' | 'system' | 'task_history' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | | 'warehouse' | 'system' | 'tasks' | 'Nullable(String)' | 'VARCHAR' | '' | '' | 'YES' | '' | diff --git a/src/query/sql/src/planner/binder/binder.rs b/src/query/sql/src/planner/binder/binder.rs index 33eb94194cdc..166cfd8e2d45 100644 --- a/src/query/sql/src/planner/binder/binder.rs +++ b/src/query/sql/src/planner/binder/binder.rs @@ -321,6 +321,7 @@ impl<'a> Binder { Statement::AlterVirtualColumn(stmt) => self.bind_alter_virtual_column(stmt).await?, Statement::DropVirtualColumn(stmt) => self.bind_drop_virtual_column(stmt).await?, Statement::RefreshVirtualColumn(stmt) => self.bind_refresh_virtual_column(stmt).await?, + Statement::ShowVirtualColumns(stmt) => self.bind_show_virtual_columns(bind_context, stmt).await?, // Users Statement::CreateUser(stmt) => self.bind_create_user(stmt).await?, diff --git a/src/query/sql/src/planner/binder/ddl/virtual_column.rs b/src/query/sql/src/planner/binder/ddl/virtual_column.rs index a5c102a96508..ed02a1243cec 100644 --- a/src/query/sql/src/planner/binder/ddl/virtual_column.rs +++ b/src/query/sql/src/planner/binder/ddl/virtual_column.rs @@ -22,17 +22,24 @@ use databend_common_ast::ast::Expr; use databend_common_ast::ast::Literal; use databend_common_ast::ast::MapAccessor; use databend_common_ast::ast::RefreshVirtualColumnStmt; +use databend_common_ast::ast::ShowLimit; +use databend_common_ast::ast::ShowVirtualColumnsStmt; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::TableDataType; use databend_common_expression::TableSchemaRef; +use log::debug; use crate::binder::Binder; +use crate::normalize_identifier; use crate::plans::AlterVirtualColumnPlan; use crate::plans::CreateVirtualColumnPlan; use crate::plans::DropVirtualColumnPlan; use crate::plans::Plan; use crate::plans::RefreshVirtualColumnPlan; +use crate::plans::RewriteKind; +use crate::BindContext; +use crate::SelectBuilder; impl Binder { #[async_backtrace::framed] @@ -41,6 +48,7 @@ impl Binder { stmt: &CreateVirtualColumnStmt, ) -> Result { let CreateVirtualColumnStmt { + if_not_exists, catalog, database, table, @@ -64,6 +72,7 @@ impl Binder { Ok(Plan::CreateVirtualColumn(Box::new( CreateVirtualColumnPlan { + if_not_exists: *if_not_exists, catalog, database, table, @@ -78,6 +87,7 @@ impl Binder { stmt: &AlterVirtualColumnStmt, ) -> Result { let AlterVirtualColumnStmt { + if_exists, catalog, database, table, @@ -100,6 +110,7 @@ impl Binder { .await?; Ok(Plan::AlterVirtualColumn(Box::new(AlterVirtualColumnPlan { + if_exists: *if_exists, catalog, database, table, @@ -113,6 +124,7 @@ impl Binder { stmt: &DropVirtualColumnStmt, ) -> Result { let DropVirtualColumnStmt { + if_exists, catalog, database, table, @@ -129,6 +141,7 @@ impl Binder { } Ok(Plan::DropVirtualColumn(Box::new(DropVirtualColumnPlan { + if_exists: *if_exists, catalog, database, table, @@ -246,4 +259,66 @@ impl Binder { Ok(virtual_columns) } + + #[async_backtrace::framed] + pub(in crate::planner::binder) async fn bind_show_virtual_columns( + &mut self, + bind_context: &mut BindContext, + stmt: &ShowVirtualColumnsStmt, + ) -> Result { + let ShowVirtualColumnsStmt { + catalog, + database, + table, + limit, + } = stmt; + + let catalog_name = match catalog { + None => self.ctx.get_current_catalog(), + Some(ident) => { + let catalog = normalize_identifier(ident, &self.name_resolution_ctx).name; + self.ctx.get_catalog(&catalog).await?; + catalog + } + }; + let catalog = self.ctx.get_catalog(&catalog_name).await?; + let database = match database { + None => self.ctx.get_current_database(), + Some(ident) => { + let database = normalize_identifier(ident, &self.name_resolution_ctx).name; + catalog + .get_database(&self.ctx.get_tenant(), &database) + .await?; + database + } + }; + + let mut select_builder = SelectBuilder::from("system.virtual_columns"); + select_builder + .with_column("database") + .with_column("table") + .with_column("virtual_columns"); + + select_builder.with_filter(format!("database = '{database}'")); + if let Some(table) = table { + let table = normalize_identifier(table, &self.name_resolution_ctx).name; + select_builder.with_filter(format!("table = '{table}'")); + } + + let query = match limit { + None => select_builder.build(), + Some(ShowLimit::Like { pattern }) => { + select_builder.with_filter(format!("virtual_columns LIKE '{pattern}'")); + select_builder.build() + } + Some(ShowLimit::Where { selection }) => { + select_builder.with_filter(format!("({selection})")); + select_builder.build() + } + }; + debug!("show virtual columns rewrite to: {:?}", query); + + self.bind_rewrite_to_query(bind_context, &query, RewriteKind::ShowVirtualColumns) + .await + } } diff --git a/src/query/sql/src/planner/plans/ddl/virtual_column.rs b/src/query/sql/src/planner/plans/ddl/virtual_column.rs index 2071858e6b85..ee0ebc253e54 100644 --- a/src/query/sql/src/planner/plans/ddl/virtual_column.rs +++ b/src/query/sql/src/planner/plans/ddl/virtual_column.rs @@ -20,6 +20,7 @@ use databend_storages_common_table_meta::meta::Location; #[derive(Clone, Debug, PartialEq, Eq)] pub struct CreateVirtualColumnPlan { + pub if_not_exists: bool, pub catalog: String, pub database: String, pub table: String, @@ -34,6 +35,7 @@ impl CreateVirtualColumnPlan { #[derive(Clone, Debug, PartialEq, Eq)] pub struct AlterVirtualColumnPlan { + pub if_exists: bool, pub catalog: String, pub database: String, pub table: String, @@ -48,6 +50,7 @@ impl AlterVirtualColumnPlan { #[derive(Clone, Debug, PartialEq, Eq)] pub struct DropVirtualColumnPlan { + pub if_exists: bool, pub catalog: String, pub database: String, pub table: String, diff --git a/src/query/sql/src/planner/plans/plan.rs b/src/query/sql/src/planner/plans/plan.rs index a0338c4b24ef..7f5366a53ee7 100644 --- a/src/query/sql/src/planner/plans/plan.rs +++ b/src/query/sql/src/planner/plans/plan.rs @@ -325,6 +325,7 @@ pub enum RewriteKind { ShowTables(String), ShowColumns(String, String), ShowTablesStatus, + ShowVirtualColumns, ShowStreams(String), diff --git a/src/query/storages/system/Cargo.toml b/src/query/storages/system/Cargo.toml index 9963edfbe10c..27afdc5eadd9 100644 --- a/src/query/storages/system/Cargo.toml +++ b/src/query/storages/system/Cargo.toml @@ -22,6 +22,7 @@ databend-common-expression = { path = "../../expression" } databend-common-functions = { path = "../../functions" } databend-common-meta-api = { path = "../../../meta/api" } databend-common-meta-app = { path = "../../../meta/app" } +databend-common-meta-types = { path = "../../../meta/types" } databend-common-metrics = { path = "../../../common/metrics" } databend-common-pipeline-core = { path = "../../pipeline/core" } databend-common-pipeline-sources = { path = "../../pipeline/sources" } diff --git a/src/query/storages/system/src/columns_table.rs b/src/query/storages/system/src/columns_table.rs index 35f09c8a84e3..6a575cfc88e9 100644 --- a/src/query/storages/system/src/columns_table.rs +++ b/src/query/storages/system/src/columns_table.rs @@ -143,87 +143,98 @@ impl ColumnsTable { ctx: Arc, push_downs: Option, ) -> Result> { - let tenant = ctx.get_tenant(); - let catalog = ctx.get_catalog(CATALOG_DEFAULT).await?; - - let mut tables = Vec::new(); - let mut databases = Vec::new(); - - if let Some(push_downs) = push_downs { - if let Some(filter) = push_downs.filters.as_ref().map(|f| &f.filter) { - let expr = filter.as_expr(&BUILTIN_FUNCTIONS); - find_eq_filter(&expr, &mut |col_name, scalar| { - if col_name == "database" { - if let Scalar::String(s) = scalar { - if let Ok(database) = String::from_utf8(s.clone()) { - if !databases.contains(&database) { - databases.push(database); - } + let database_and_tables = dump_tables(&ctx, push_downs).await?; + + let mut rows: Vec<(String, String, TableField)> = vec![]; + for (database, tables) in database_and_tables { + for table in tables { + let fields = generate_fields(&ctx, &table).await?; + for field in fields { + rows.push((database.clone(), table.name().into(), field.clone())) + } + } + } + + Ok(rows) + } +} + +pub(crate) async fn dump_tables( + ctx: &Arc, + push_downs: Option, +) -> Result>)>> { + let tenant = ctx.get_tenant(); + let catalog = ctx.get_catalog(CATALOG_DEFAULT).await?; + + let mut tables = Vec::new(); + let mut databases = Vec::new(); + + if let Some(push_downs) = push_downs { + if let Some(filter) = push_downs.filters.as_ref().map(|f| &f.filter) { + let expr = filter.as_expr(&BUILTIN_FUNCTIONS); + find_eq_filter(&expr, &mut |col_name, scalar| { + if col_name == "database" { + if let Scalar::String(s) = scalar { + if let Ok(database) = String::from_utf8(s.clone()) { + if !databases.contains(&database) { + databases.push(database); } } - } else if col_name == "table" { - if let Scalar::String(s) = scalar { - if let Ok(table) = String::from_utf8(s.clone()) { - if !tables.contains(&table) { - tables.push(table); - } + } + } else if col_name == "table" { + if let Scalar::String(s) = scalar { + if let Ok(table) = String::from_utf8(s.clone()) { + if !tables.contains(&table) { + tables.push(table); } } } - }); - } + } + }); } + } - if databases.is_empty() { - let all_databases = catalog.list_databases(tenant.as_str()).await?; - for db in all_databases { - databases.push(db.name().to_string()); - } + if databases.is_empty() { + let all_databases = catalog.list_databases(tenant.as_str()).await?; + for db in all_databases { + databases.push(db.name().to_string()); } + } - let tenant = ctx.get_tenant(); - let visibility_checker = ctx.get_visibility_checker().await?; + let visibility_checker = ctx.get_visibility_checker().await?; - let final_dbs: Vec = databases - .iter() - .filter(|db| visibility_checker.check_database_visibility(CATALOG_DEFAULT, db)) - .cloned() - .collect(); + let final_dbs: Vec = databases + .iter() + .filter(|db| visibility_checker.check_database_visibility(CATALOG_DEFAULT, db)) + .cloned() + .collect(); - let mut rows: Vec<(String, String, TableField)> = vec![]; - for database in final_dbs { - let tables = if tables.is_empty() { - if let Ok(table) = catalog.list_tables(tenant.as_str(), &database).await { - table - } else { - vec![] - } + let mut final_tables: Vec<(String, Vec>)> = Vec::with_capacity(final_dbs.len()); + for database in final_dbs { + let tables = if tables.is_empty() { + if let Ok(table) = catalog.list_tables(tenant.as_str(), &database).await { + table } else { - let mut res = Vec::new(); - for table in &tables { - if let Ok(table) = catalog.get_table(tenant.as_str(), &database, table).await { - res.push(table); - } - } - res - }; - - for table in tables { - if visibility_checker.check_table_visibility( - CATALOG_DEFAULT, - &database, - table.name(), - ) { - let fields = generate_fields(&ctx, &table).await?; - for field in fields { - rows.push((database.clone(), table.name().into(), field.clone())) - } + vec![] + } + } else { + let mut res = Vec::new(); + for table in &tables { + if let Ok(table) = catalog.get_table(tenant.as_str(), &database, table).await { + res.push(table); } } + res + }; + let mut filtered_tables = Vec::with_capacity(tables.len()); + for table in tables { + if visibility_checker.check_table_visibility(CATALOG_DEFAULT, &database, table.name()) { + filtered_tables.push(table); + } } - - Ok(rows) + final_tables.push((database, filtered_tables)); } + Ok(final_tables) } async fn generate_fields( diff --git a/src/query/storages/system/src/lib.rs b/src/query/storages/system/src/lib.rs index bcf3fe7e17f2..977884955a9b 100644 --- a/src/query/storages/system/src/lib.rs +++ b/src/query/storages/system/src/lib.rs @@ -59,6 +59,7 @@ mod temp_files_table; mod tracing_table; mod users_table; mod util; +mod virtual_columns_table; pub use background_jobs_table::BackgroundJobTable; pub use background_tasks_table::BackgroundTaskTable; @@ -112,3 +113,4 @@ pub use tasks_table::TasksTable; pub use temp_files_table::TempFilesTable; pub use tracing_table::TracingTable; pub use users_table::UsersTable; +pub use virtual_columns_table::VirtualColumnsTable; diff --git a/src/query/storages/system/src/virtual_columns_table.rs b/src/query/storages/system/src/virtual_columns_table.rs new file mode 100644 index 000000000000..1dbf511d5d31 --- /dev/null +++ b/src/query/storages/system/src/virtual_columns_table.rs @@ -0,0 +1,138 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; +use std::sync::Arc; + +use databend_common_catalog::catalog::CATALOG_DEFAULT; +use databend_common_catalog::plan::PushDownInfo; +use databend_common_catalog::table::Table; +use databend_common_exception::Result; +use databend_common_expression::types::StringType; +use databend_common_expression::types::TimestampType; +use databend_common_expression::DataBlock; +use databend_common_expression::FromData; +use databend_common_expression::TableDataType; +use databend_common_expression::TableField; +use databend_common_expression::TableSchemaRefExt; +use databend_common_meta_app::schema::ListVirtualColumnsReq; +use databend_common_meta_app::schema::TableIdent; +use databend_common_meta_app::schema::TableInfo; +use databend_common_meta_app::schema::TableMeta; +use databend_common_meta_app::schema::VirtualColumnMeta; +use databend_common_meta_types::MetaId; +use databend_common_storages_fuse::TableContext; + +use crate::columns_table::dump_tables; +use crate::table::AsyncOneBlockSystemTable; +use crate::table::AsyncSystemTable; + +pub struct VirtualColumnsTable { + table_info: TableInfo, +} + +#[async_trait::async_trait] +impl AsyncSystemTable for VirtualColumnsTable { + const NAME: &'static str = "system.virtual_columns"; + + fn get_table_info(&self) -> &TableInfo { + &self.table_info + } + + async fn get_full_data( + &self, + ctx: Arc, + push_downs: Option, + ) -> Result { + let tenant = ctx.get_tenant(); + let catalog = ctx.get_catalog(CATALOG_DEFAULT).await?; + let virtual_column_metas = catalog + .list_virtual_columns(ListVirtualColumnsReq { + tenant: tenant.clone(), + table_id: None, + }) + .await?; + + let mut database_names = Vec::with_capacity(virtual_column_metas.len()); + let mut table_names = Vec::with_capacity(virtual_column_metas.len()); + let mut virtual_columns = Vec::with_capacity(virtual_column_metas.len()); + let mut created_ons = Vec::with_capacity(virtual_column_metas.len()); + let mut updated_ons = Vec::with_capacity(virtual_column_metas.len()); + if !virtual_column_metas.is_empty() { + let mut virtual_column_meta_map: HashMap = + virtual_column_metas + .into_iter() + .map(|v| (v.table_id, v)) + .collect(); + + let database_and_tables = dump_tables(&ctx, push_downs).await?; + for (database, tables) in database_and_tables { + for table in tables { + let table_id = table.get_id(); + if let Some(virtual_column_meta) = virtual_column_meta_map.remove(&table_id) { + database_names.push(database.as_bytes().to_vec()); + table_names.push(table.name().as_bytes().to_vec()); + virtual_columns.push( + virtual_column_meta + .virtual_columns + .join(", ") + .as_bytes() + .to_vec(), + ); + created_ons.push(virtual_column_meta.created_on.timestamp_micros()); + updated_ons + .push(virtual_column_meta.updated_on.map(|u| u.timestamp_micros())); + } + } + } + } + + Ok(DataBlock::new_from_columns(vec![ + StringType::from_data(database_names), + StringType::from_data(table_names), + StringType::from_data(virtual_columns), + TimestampType::from_data(created_ons), + TimestampType::from_opt_data(updated_ons), + ])) + } +} + +impl VirtualColumnsTable { + pub fn create(table_id: u64) -> Arc { + let schema = TableSchemaRefExt::create(vec![ + TableField::new("database", TableDataType::String), + TableField::new("table", TableDataType::String), + TableField::new("virtual_columns", TableDataType::String), + TableField::new("created_on", TableDataType::Timestamp), + TableField::new( + "updated_on", + TableDataType::Nullable(Box::new(TableDataType::Timestamp)), + ), + ]); + + let table_info = TableInfo { + desc: "'system'.'virtual_columns'".to_string(), + name: "virtual_columns".to_string(), + ident: TableIdent::new(table_id, 0), + meta: TableMeta { + schema, + engine: "SystemVirtualColumns".to_string(), + ..Default::default() + }, + ..Default::default() + }; + + AsyncOneBlockSystemTable::create(Self { table_info }) + } +} diff --git a/tests/sqllogictests/suites/ee/01_ee_system/01_0002_virtual_column.test b/tests/sqllogictests/suites/ee/01_ee_system/01_0002_virtual_column.test index c819ec4146ef..f912e7715d6d 100644 --- a/tests/sqllogictests/suites/ee/01_ee_system/01_0002_virtual_column.test +++ b/tests/sqllogictests/suites/ee/01_ee_system/01_0002_virtual_column.test @@ -36,6 +36,11 @@ create virtual column (val['a'], val['b']) for t1 statement ok refresh virtual column for t1 +query TTT +show virtual columns from t1 +---- +test_virtual_column t1 val['a'], val['b'] + statement ok insert into t1 values(4, '{"a":44,"b":4}'), (5, '{"a":55}'), (6, '6') @@ -135,6 +140,11 @@ create virtual column (val['a'], val['b']) for t2 statement ok refresh virtual column for t2 +query TTT +show virtual columns from t2 +---- +test_virtual_column t2 val['a'], val['b'] + statement ok insert into t2 values(4, '{"a":44,"b":4}'), (5, '{"a":55}'), (6, '6') diff --git a/tests/sqllogictests/suites/ee/05_ee_ddl/05_0002_ddl_create_drop_virtual_columns.test b/tests/sqllogictests/suites/ee/05_ee_ddl/05_0002_ddl_create_drop_virtual_columns.test index 3b23cc3b314b..77dcc2e5737c 100644 --- a/tests/sqllogictests/suites/ee/05_ee_ddl/05_0002_ddl_create_drop_virtual_columns.test +++ b/tests/sqllogictests/suites/ee/05_ee_ddl/05_0002_ddl_create_drop_virtual_columns.test @@ -30,9 +30,15 @@ CREATE TABLE t1(a int, v json) Engine = Fuse statement error 1115 ALTER VIRTUAL COLUMN (v['k1'], v:k2, v[0]) FOR t1; +statement ok +ALTER VIRTUAL COLUMN IF EXISTS (v['k1'], v:k2) FOR t1; + statement error 1115 DROP VIRTUAL COLUMN FOR t1; +statement ok +DROP VIRTUAL COLUMN IF EXISTS FOR t1; + statement error 1065 CREATE VIRTUAL COLUMN (v) FOR t1; @@ -42,6 +48,9 @@ CREATE VIRTUAL COLUMN (a['k1']) FOR t1; statement ok CREATE VIRTUAL COLUMN (v['k1'], v:k2, v[0]) FOR t1; +statement ok +CREATE VIRTUAL COLUMN IF NOT EXISTS (v['k1'], v:k2, v[0]) FOR t1; + statement error 1116 CREATE VIRTUAL COLUMN (v['k1'], v:k2, v[0]) FOR t1; From 4705cae1b8edc7069d76205a43232c508343e6a1 Mon Sep 17 00:00:00 2001 From: sundyli <543950155@qq.com> Date: Tue, 19 Dec 2023 02:20:10 -0800 Subject: [PATCH 18/20] chore(query): refactor function params to be scalar (#14079) * chore(query): refactor function params to be scalar * chore(query): refactor function params to be scalar --- src/query/expression/src/evaluator.rs | 11 +- src/query/expression/src/expression.rs | 2 +- src/query/expression/src/function.rs | 10 +- src/query/expression/src/type_check.rs | 12 +- src/query/expression/src/types/decimal.rs | 11 ++ src/query/expression/src/types/number.rs | 8 ++ src/query/expression/src/values.rs | 17 +++ src/query/functions/src/scalars/arithmetic.rs | 124 +++--------------- .../functions/src/scalars/decimal/cast.rs | 75 ++++++++++- .../functions/src/scalars/decimal/math.rs | 6 +- .../functions/src/scalars/decimal/mod.rs | 1 + src/query/functions/src/scalars/other.rs | 3 +- src/query/functions/src/scalars/tuple.rs | 7 +- src/query/functions/src/srfs/variant.rs | 4 +- .../functions/tests/it/aggregates/mod.rs | 6 - .../functions/tests/it/scalars/parser.rs | 10 +- .../tests/it/scalars/testdata/math.txt | 18 +-- src/query/sql/src/planner/binder/aggregate.rs | 2 +- src/query/sql/src/planner/binder/table.rs | 2 +- .../planner/optimizer/rule/utils/constant.rs | 2 +- .../sql/src/planner/plans/scalar_expr.rs | 2 +- .../sql/src/planner/semantic/lowering.rs | 7 +- .../sql/src/planner/semantic/type_check.rs | 16 +-- 23 files changed, 190 insertions(+), 166 deletions(-) diff --git a/src/query/expression/src/evaluator.rs b/src/query/expression/src/evaluator.rs index 82145643d517..37a7c06a7df5 100644 --- a/src/query/expression/src/evaluator.rs +++ b/src/query/expression/src/evaluator.rs @@ -38,6 +38,7 @@ use crate::types::nullable::NullableDomain; use crate::types::BooleanType; use crate::types::DataType; use crate::types::NullableType; +use crate::types::NumberScalar; use crate::values::Column; use crate::values::ColumnBuilder; use crate::values::Scalar; @@ -750,7 +751,10 @@ impl<'a> Evaluator<'a> { }; let params = if let DataType::Decimal(ty) = dest_type.remove_nullable() { - vec![ty.precision() as usize, ty.scale() as usize] + vec![ + Scalar::Number(NumberScalar::Int64(ty.precision() as _)), + Scalar::Number(NumberScalar::Int64(ty.scale() as _)), + ] } else { vec![] }; @@ -1618,7 +1622,10 @@ impl<'a, Index: ColumnIndex> ConstantFolder<'a, Index> { }; let params = if let DataType::Decimal(ty) = dest_type { - vec![ty.precision() as usize, ty.scale() as usize] + vec![ + Scalar::Number(NumberScalar::Int64(ty.precision() as _)), + Scalar::Number(NumberScalar::Int64(ty.scale() as _)), + ] } else { vec![] }; diff --git a/src/query/expression/src/expression.rs b/src/query/expression/src/expression.rs index 9cab633d49c1..1a32ae332490 100644 --- a/src/query/expression/src/expression.rs +++ b/src/query/expression/src/expression.rs @@ -60,7 +60,7 @@ pub enum RawExpr { FunctionCall { span: Span, name: String, - params: Vec, + params: Vec, args: Vec>, }, LambdaFunctionCall { diff --git a/src/query/expression/src/function.rs b/src/query/expression/src/function.rs index cc9aa9b4ca13..592b742cae51 100755 --- a/src/query/expression/src/function.rs +++ b/src/query/expression/src/function.rs @@ -49,7 +49,7 @@ pub type AutoCastRules<'a> = &'a [(DataType, DataType)]; /// /// The first argument is the const parameters and the second argument is the types of arguments. pub trait FunctionFactory = - Fn(&[usize], &[DataType]) -> Option> + Send + Sync + 'static; + Fn(&[Scalar], &[DataType]) -> Option> + Send + Sync + 'static; pub struct Function { pub signature: FunctionSignature, @@ -131,7 +131,7 @@ pub enum FunctionID { Factory { name: String, id: usize, - params: Vec, + params: Vec, args_type: Vec, }, } @@ -327,7 +327,7 @@ impl FunctionRegistry { pub fn search_candidates( &self, name: &str, - params: &[usize], + params: &[Scalar], args: &[Expr], ) -> Vec<(FunctionID, Arc)> { let name = name.to_lowercase(); @@ -522,7 +522,7 @@ impl FunctionID { } } - pub fn params(&self) -> &[usize] { + pub fn params(&self) -> &[Scalar] { match self { FunctionID::Builtin { .. } => &[], FunctionID::Factory { params, .. } => params.as_slice(), @@ -558,7 +558,7 @@ impl<'a> EvalContext<'a> { pub fn render_error( &self, span: Span, - params: &[usize], + params: &[Scalar], args: &[Value], func_name: &str, ) -> Result<()> { diff --git a/src/query/expression/src/type_check.rs b/src/query/expression/src/type_check.rs index d1b1f887cb92..9fdcfc434022 100755 --- a/src/query/expression/src/type_check.rs +++ b/src/query/expression/src/type_check.rs @@ -143,13 +143,12 @@ pub fn check( } else { new_args.push(Expr::Constant { span: None, - scalar: Scalar::Number(NumberScalar::Int64(scale)), + scalar: Scalar::Number(scale.into()), data_type: Int64Type::data_type(), }) } scale = scale.clamp(-76, 76); - let add_on_scale = (scale + 76) as usize; - let params = vec![add_on_scale]; + let params = vec![Scalar::Number(scale.into())]; return check_function(*span, name, ¶ms, &args_expr, fn_registry); } @@ -206,7 +205,10 @@ pub fn check_cast( // fast path to eval function for cast if let Some(cast_fn) = get_simple_cast_function(is_try, dest_type) { let params = if let DataType::Decimal(ty) = dest_type { - vec![ty.precision() as usize, ty.scale() as usize] + vec![ + Scalar::Number(NumberScalar::Int64(ty.precision() as _)), + Scalar::Number(NumberScalar::Int64(ty.scale() as _)), + ] } else { vec![] }; @@ -286,7 +288,7 @@ pub fn check_number( pub fn check_function( span: Span, name: &str, - params: &[usize], + params: &[Scalar], args: &[Expr], fn_registry: &FunctionRegistry, ) -> Result> { diff --git a/src/query/expression/src/types/decimal.rs b/src/query/expression/src/types/decimal.rs index 91233032fe16..4c5b19697fb3 100644 --- a/src/query/expression/src/types/decimal.rs +++ b/src/query/expression/src/types/decimal.rs @@ -21,6 +21,8 @@ use borsh::BorshSerialize; use databend_common_arrow::arrow::buffer::Buffer; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_io::display_decimal_128; +use databend_common_io::display_decimal_256; use enum_as_inner::EnumAsInner; use ethnum::i256; use ethnum::AsI256; @@ -307,6 +309,7 @@ pub trait Decimal: fn from_i128>(value: U) -> Self; fn de_binary(bytes: &mut &[u8]) -> Self; + fn display(self, scale: u8) -> String; fn to_float32(self, scale: u8) -> f32; fn to_float64(self, scale: u8) -> f64; @@ -453,6 +456,10 @@ impl Decimal for i128 { i128::from_le_bytes(bs) } + fn display(self, scale: u8) -> String { + display_decimal_128(self, scale) + } + fn to_float32(self, scale: u8) -> f32 { let div = 10_f32.powi(scale as i32); self as f32 / div @@ -618,6 +625,10 @@ impl Decimal for i256 { i256::from_le_bytes(bs) } + fn display(self, scale: u8) -> String { + display_decimal_256(self, scale) + } + fn to_float32(self, scale: u8) -> f32 { let div = 10_f32.powi(scale as i32); self.as_f32() / div diff --git a/src/query/expression/src/types/number.rs b/src/query/expression/src/types/number.rs index 58fe6a919dca..93f8902c2271 100644 --- a/src/query/expression/src/types/number.rs +++ b/src/query/expression/src/types/number.rs @@ -502,6 +502,14 @@ impl NumberScalar { } } +impl From for NumberScalar +where T: Number +{ + fn from(value: T) -> Self { + T::upcast_scalar(value) + } +} + impl NumberColumn { pub fn len(&self) -> usize { crate::with_number_type!(|NUM_TYPE| match self { diff --git a/src/query/expression/src/values.rs b/src/query/expression/src/values.rs index 2a43d27c6767..aedc7aaa8478 100755 --- a/src/query/expression/src/values.rs +++ b/src/query/expression/src/values.rs @@ -408,6 +408,23 @@ impl Scalar { _ => unreachable!("is_positive() called on non-numeric scalar"), } } + + pub fn get_i64(&self) -> Option { + match self { + Scalar::Number(n) => match n { + NumberScalar::Int8(x) => Some(*x as _), + NumberScalar::Int16(x) => Some(*x as _), + NumberScalar::Int32(x) => Some(*x as _), + NumberScalar::Int64(x) => Some(*x as _), + NumberScalar::UInt8(x) => Some(*x as _), + NumberScalar::UInt16(x) => Some(*x as _), + NumberScalar::UInt32(x) => Some(*x as _), + NumberScalar::UInt64(x) => i64::try_from(*x).ok(), + _ => None, + }, + _ => None, + } + } } impl<'a> ScalarRef<'a> { diff --git a/src/query/functions/src/scalars/arithmetic.rs b/src/query/functions/src/scalars/arithmetic.rs index 1319dacc70e6..3958ddd1a192 100644 --- a/src/query/functions/src/scalars/arithmetic.rs +++ b/src/query/functions/src/scalars/arithmetic.rs @@ -20,9 +20,8 @@ use std::ops::BitXor; use std::sync::Arc; use databend_common_arrow::arrow::bitmap::Bitmap; -use databend_common_expression::types::decimal::Decimal; -use databend_common_expression::types::decimal::DecimalColumn; use databend_common_expression::types::decimal::DecimalDomain; +use databend_common_expression::types::decimal::DecimalType; use databend_common_expression::types::nullable::NullableColumn; use databend_common_expression::types::nullable::NullableDomain; use databend_common_expression::types::number::Number; @@ -30,7 +29,6 @@ use databend_common_expression::types::number::NumberType; use databend_common_expression::types::number::F64; use databend_common_expression::types::string::StringColumnBuilder; use databend_common_expression::types::AnyType; -use databend_common_expression::types::ArgType; use databend_common_expression::types::DataType; use databend_common_expression::types::DecimalDataType; use databend_common_expression::types::NullableType; @@ -51,13 +49,12 @@ use databend_common_expression::values::ValueRef; use databend_common_expression::vectorize_1_arg; use databend_common_expression::vectorize_with_builder_1_arg; use databend_common_expression::vectorize_with_builder_2_arg; +use databend_common_expression::with_decimal_mapped_type; use databend_common_expression::with_float_mapped_type; use databend_common_expression::with_integer_mapped_type; use databend_common_expression::with_number_mapped_type; use databend_common_expression::with_number_mapped_type_without_64; use databend_common_expression::with_unsigned_integer_mapped_type; -use databend_common_expression::Column; -use databend_common_expression::ColumnBuilder; use databend_common_expression::Domain; use databend_common_expression::EvalContext; use databend_common_expression::Function; @@ -65,9 +62,6 @@ use databend_common_expression::FunctionDomain; use databend_common_expression::FunctionEval; use databend_common_expression::FunctionRegistry; use databend_common_expression::FunctionSignature; -use databend_common_expression::Scalar; -use databend_common_io::display_decimal_128; -use databend_common_io::display_decimal_256; use ethnum::i256; use lexical_core::FormattedSize; use num_traits::AsPrimitive; @@ -76,6 +70,7 @@ use super::arithmetic_modulo::vectorize_modulo; use super::decimal::register_decimal_to_int; use crate::scalars::decimal::register_decimal_arithmetic; use crate::scalars::decimal::register_decimal_to_float; +use crate::scalars::decimal::register_decimal_to_string; pub fn register(registry: &mut FunctionRegistry) { registry.register_aliases("plus", &["add"]); @@ -775,7 +770,7 @@ pub fn register_decimal_minus(registry: &mut FunctionRegistry) { } _ => unreachable!(), }), - eval: Box::new(move |args, _tx| unary_minus_decimal(args, arg_type.clone())), + eval: Box::new(move |args, ctx| unary_minus_decimal(args, arg_type.clone(), ctx)), }, }; @@ -787,34 +782,20 @@ pub fn register_decimal_minus(registry: &mut FunctionRegistry) { }); } -fn unary_minus_decimal(args: &[ValueRef], arg_type: DataType) -> Value { +fn unary_minus_decimal( + args: &[ValueRef], + arg_type: DataType, + ctx: &mut EvalContext, +) -> Value { let arg = &args[0]; - let mut is_scalar = false; - let column = match arg { - ValueRef::Column(column) => column.clone(), - ValueRef::Scalar(s) => { - is_scalar = true; - let builder = ColumnBuilder::repeat(s, 1, &arg_type); - builder.build() - } - }; - - let result = match column { - Column::Decimal(DecimalColumn::Decimal128(buf, size)) => { - DecimalColumn::Decimal128(buf.into_iter().map(|x| -x).collect(), size) - } - Column::Decimal(DecimalColumn::Decimal256(buf, size)) => { - DecimalColumn::Decimal256(buf.into_iter().map(|x| -x).collect(), size) + let arg_type = arg_type.as_decimal().unwrap(); + with_decimal_mapped_type!(|DECIMAL_TYPE| match arg_type { + DecimalDataType::DECIMAL_TYPE(size) => { + type Type = DecimalType; + let arg = arg.try_downcast().unwrap(); + vectorize_1_arg::(|t, _| -t)(arg, ctx).upcast_decimal(*size) } - _ => unreachable!(), - }; - - if is_scalar { - let scalar = result.index(0).unwrap(); - Value::Scalar(Scalar::Decimal(scalar)) - } else { - Value::Column(Column::Decimal(result)) - } + }) } fn register_string_to_number(registry: &mut FunctionRegistry) { @@ -953,76 +934,3 @@ pub fn register_number_to_string(registry: &mut FunctionRegistry) { }); } } - -fn register_decimal_to_string(registry: &mut FunctionRegistry) { - // decimal to string - registry.register_function_factory("to_string", |_params, args_type| { - if args_type.len() != 1 { - return None; - } - - let arg_type = args_type[0].remove_nullable(); - if !arg_type.is_decimal() { - return None; - } - - Some(Arc::new(Function { - signature: FunctionSignature { - name: "to_string".to_string(), - args_type: vec![arg_type.clone()], - return_type: StringType::data_type(), - }, - eval: FunctionEval::Scalar { - calc_domain: Box::new(|_, _| FunctionDomain::Full), - eval: Box::new(move |args, tx| decimal_to_string(args, arg_type.clone(), tx)), - }, - })) - }); -} - -fn decimal_to_string( - args: &[ValueRef], - from_type: DataType, - _ctx: &mut EvalContext, -) -> Value { - let arg = &args[0]; - - let mut is_scalar = false; - let column = match arg { - ValueRef::Column(column) => column.clone(), - ValueRef::Scalar(s) => { - is_scalar = true; - let builder = ColumnBuilder::repeat(s, 1, &from_type); - builder.build() - } - }; - - let from_type = from_type.as_decimal().unwrap(); - - let column = match from_type { - DecimalDataType::Decimal128(_) => { - let (buffer, from_size) = i128::try_downcast_column(&column).unwrap(); - let mut builder = StringColumnBuilder::with_capacity(buffer.len(), buffer.len() * 10); - for x in buffer { - builder.put_str(&display_decimal_128(x, from_size.scale)); - builder.commit_row(); - } - builder - } - DecimalDataType::Decimal256(_) => { - let (buffer, from_size) = i256::try_downcast_column(&column).unwrap(); - let mut builder = StringColumnBuilder::with_capacity(buffer.len(), buffer.len() * 10); - for x in buffer { - builder.put_str(&display_decimal_256(x, from_size.scale)); - builder.commit_row(); - } - builder - } - }; - - if is_scalar { - Value::Scalar(Scalar::String(column.build_scalar())) - } else { - Value::Column(Column::String(column.build())) - } -} diff --git a/src/query/functions/src/scalars/decimal/cast.rs b/src/query/functions/src/scalars/decimal/cast.rs index 58b3d9d83d60..e1c4e9b3168a 100644 --- a/src/query/functions/src/scalars/decimal/cast.rs +++ b/src/query/functions/src/scalars/decimal/cast.rs @@ -17,12 +17,14 @@ use std::sync::Arc; use databend_common_expression::serialize::read_decimal_with_size; use databend_common_expression::types::decimal::*; +use databend_common_expression::types::string::StringColumnBuilder; use databend_common_expression::types::*; use databend_common_expression::vectorize_1_arg; use databend_common_expression::vectorize_with_builder_1_arg; use databend_common_expression::with_decimal_mapped_type; use databend_common_expression::with_integer_mapped_type; use databend_common_expression::with_number_mapped_type; +use databend_common_expression::Column; use databend_common_expression::Domain; use databend_common_expression::EvalContext; use databend_common_expression::FromData; @@ -32,6 +34,7 @@ use databend_common_expression::FunctionDomain; use databend_common_expression::FunctionEval; use databend_common_expression::FunctionRegistry; use databend_common_expression::FunctionSignature; +use databend_common_expression::Scalar; use databend_common_expression::Value; use databend_common_expression::ValueRef; use ethnum::i256; @@ -40,7 +43,7 @@ use ordered_float::OrderedFloat; // int float to decimal pub fn register_to_decimal(registry: &mut FunctionRegistry) { - let factory = |params: &[usize], args_type: &[DataType]| { + let factory = |params: &[Scalar], args_type: &[DataType]| { if args_type.len() != 1 { return None; } @@ -58,8 +61,8 @@ pub fn register_to_decimal(registry: &mut FunctionRegistry) { } let decimal_size = DecimalSize { - precision: params[0] as u8, - scale: params[1] as u8, + precision: params[0].get_i64()? as _, + scale: params[1].get_i64()? as _, }; let decimal_type = DecimalDataType::from_size(decimal_size).ok()?; @@ -108,7 +111,7 @@ pub(crate) fn register_decimal_to_float(registry: &mut FunctionRegist let is_f32 = matches!(data_type, DataType::Number(NumberDataType::Float32)); - let factory = |_params: &[usize], args_type: &[DataType], data_type: DataType| { + let factory = |_params: &[Scalar], args_type: &[DataType], data_type: DataType| { if args_type.len() != 1 { return None; } @@ -203,7 +206,7 @@ pub(crate) fn register_decimal_to_int(registry: &mut FunctionRegistry let name = format!("to_{}", T::data_type().to_string().to_lowercase()); let try_name = format!("try_to_{}", T::data_type().to_string().to_lowercase()); - let factory = |_params: &[usize], args_type: &[DataType]| { + let factory = |_params: &[Scalar], args_type: &[DataType]| { if args_type.len() != 1 { return None; } @@ -263,6 +266,68 @@ pub(crate) fn register_decimal_to_int(registry: &mut FunctionRegistry }); } +pub(crate) fn register_decimal_to_string(registry: &mut FunctionRegistry) { + // decimal to string + let factory = |_params: &[Scalar], args_type: &[DataType]| { + if args_type.len() != 1 { + return None; + } + + let arg_type = args_type[0].remove_nullable(); + if !arg_type.is_decimal() { + return None; + } + + let function = Function { + signature: FunctionSignature { + name: "to_string".to_string(), + args_type: vec![arg_type.clone()], + return_type: StringType::data_type(), + }, + eval: FunctionEval::Scalar { + calc_domain: Box::new(|_, _| FunctionDomain::Full), + eval: Box::new(move |args, tx| decimal_to_string(args, arg_type.clone(), tx)), + }, + }; + + if args_type[0].is_nullable() { + Some(Arc::new(function.passthrough_nullable())) + } else { + Some(Arc::new(function)) + } + }; + registry.register_function_factory("to_string", factory); +} + +fn decimal_to_string( + args: &[ValueRef], + from_type: DataType, + _ctx: &mut EvalContext, +) -> Value { + let arg = &args[0]; + let from_type = from_type.as_decimal().unwrap(); + + with_decimal_mapped_type!(|DECIMAL_TYPE| match from_type { + DecimalDataType::DECIMAL_TYPE(from_size) => { + let arg: ValueRef> = arg.try_downcast().unwrap(); + + match arg { + ValueRef::Column(col) => { + let mut builder = StringColumnBuilder::with_capacity(col.len(), col.len() * 10); + for x in DecimalType::::iter_column(&col) { + builder.put_str(&DECIMAL_TYPE::display(x, from_size.scale)); + builder.commit_row(); + } + Value::Column(Column::String(builder.build())) + } + ValueRef::Scalar(x) => Value::Scalar(Scalar::String( + DECIMAL_TYPE::display(x, from_size.scale).into(), + )), + } + } + }) +} + fn convert_to_decimal( arg: &ValueRef, ctx: &mut EvalContext, diff --git a/src/query/functions/src/scalars/decimal/math.rs b/src/query/functions/src/scalars/decimal/math.rs index 7ed5976b858b..886785dfcc5e 100644 --- a/src/query/functions/src/scalars/decimal/math.rs +++ b/src/query/functions/src/scalars/decimal/math.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::cmp::Ord; use std::ops::*; use std::sync::Arc; @@ -26,12 +25,13 @@ use databend_common_expression::FunctionDomain; use databend_common_expression::FunctionEval; use databend_common_expression::FunctionRegistry; use databend_common_expression::FunctionSignature; +use databend_common_expression::Scalar; use databend_common_expression::Value; use databend_common_expression::ValueRef; use ethnum::i256; pub fn register_decimal_math(registry: &mut FunctionRegistry) { - let factory = |params: &[usize], args_type: &[DataType], round_mode: RoundMode| { + let factory = |params: &[Scalar], args_type: &[DataType], round_mode: RoundMode| { if args_type.is_empty() { return None; } @@ -46,7 +46,7 @@ pub fn register_decimal_math(registry: &mut FunctionRegistry) { let scale = if params.is_empty() { 0 } else { - params[0] as i64 - 76 + params[0].get_i64()? }; let decimal_size = DecimalSize { diff --git a/src/query/functions/src/scalars/decimal/mod.rs b/src/query/functions/src/scalars/decimal/mod.rs index 12407ca97278..c55ea2d36f56 100644 --- a/src/query/functions/src/scalars/decimal/mod.rs +++ b/src/query/functions/src/scalars/decimal/mod.rs @@ -20,6 +20,7 @@ mod math; pub(crate) use arithmetic::register_decimal_arithmetic; pub(crate) use cast::register_decimal_to_float; pub(crate) use cast::register_decimal_to_int; +pub(crate) use cast::register_decimal_to_string; pub(crate) use cast::register_to_decimal; pub(crate) use comparison::register_decimal_compare_op; pub(crate) use math::register_decimal_math; diff --git a/src/query/functions/src/scalars/other.rs b/src/query/functions/src/scalars/other.rs index 85287d5070a8..46c02e2ca2fa 100644 --- a/src/query/functions/src/scalars/other.rs +++ b/src/query/functions/src/scalars/other.rs @@ -349,7 +349,8 @@ fn register_grouping(registry: &mut FunctionRegistry) { return None; } - let params = params.to_vec(); + let params: Vec = params.iter().map(|p| p.get_i64().unwrap() as _).collect(); + Some(Arc::new(Function { signature: FunctionSignature { name: "grouping".to_string(), diff --git a/src/query/functions/src/scalars/tuple.rs b/src/query/functions/src/scalars/tuple.rs index c3ae24e0eb55..00aa6df387db 100644 --- a/src/query/functions/src/scalars/tuple.rs +++ b/src/query/functions/src/scalars/tuple.rs @@ -81,7 +81,8 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_function_factory("get", |params, args_type| { // Tuple index starts from 1 - let idx = params.first()?.checked_sub(1)?; + let idx = (params.first()?.get_i64()? as usize).checked_sub(1)?; + let fields_ty = match args_type.first()? { DataType::Tuple(tys) => tys, _ => return None, @@ -117,7 +118,7 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_function_factory("get", |params, args_type| { // Tuple index starts from 1 - let idx = params.first()?.checked_sub(1)?; + let idx = usize::try_from(params.first()?.get_i64()? - 1).ok()?; let fields_ty = match args_type.first()? { DataType::Nullable(box DataType::Tuple(tys)) => tys, _ => return None, @@ -179,7 +180,7 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_function_factory("get", |params, args_type| { // Tuple index starts from 1 - let idx = params.first()?.checked_sub(1)?; + let idx = usize::try_from(params.first()?.get_i64()? - 1).ok()?; let fields_ty = match args_type.first()? { DataType::Nullable(box DataType::Tuple(tys)) => tys, _ => return None, diff --git a/src/query/functions/src/srfs/variant.rs b/src/query/functions/src/srfs/variant.rs index 606616f91761..6bd1a1cf25ea 100644 --- a/src/query/functions/src/srfs/variant.rs +++ b/src/query/functions/src/srfs/variant.rs @@ -269,7 +269,7 @@ pub fn register(registry: &mut FunctionRegistry) { { return None; } - let params = params.to_vec(); + let params: Vec = params.iter().map(|x| x.get_i64().unwrap()).collect(); Some(Arc::new(Function { signature: FunctionSignature { @@ -707,7 +707,7 @@ impl FlattenGenerator { } } - fn generate(&mut self, seq: u64, input: &[u8], path: &str, params: &[usize]) -> Vec { + fn generate(&mut self, seq: u64, input: &[u8], path: &str, params: &[i64]) -> Vec { // Only columns required by parent plan need a builder. let mut key_builder = if params.is_empty() || params.contains(&2) { Some(NullableColumnBuilder::::with_capacity(0, &[])) diff --git a/src/query/functions/tests/it/aggregates/mod.rs b/src/query/functions/tests/it/aggregates/mod.rs index 37f80a146def..b25935554969 100644 --- a/src/query/functions/tests/it/aggregates/mod.rs +++ b/src/query/functions/tests/it/aggregates/mod.rs @@ -21,7 +21,6 @@ use bumpalo::Bump; use comfy_table::Table; use databend_common_exception::Result; use databend_common_expression::type_check; -use databend_common_expression::types::number::NumberScalar; use databend_common_expression::types::AnyType; use databend_common_expression::types::DataType; use databend_common_expression::BlockEntry; @@ -85,11 +84,6 @@ pub fn run_agg_ast( .collect::>() .unwrap(); - let params = params - .iter() - .map(|p| Scalar::Number(NumberScalar::UInt64(*p as u64))) - .collect(); - // Convert the delimiter of string_agg to params let params = if name.eq_ignore_ascii_case("string_agg") && args.len() == 2 { let val = args[1].0.as_scalar().unwrap(); diff --git a/src/query/functions/tests/it/scalars/parser.rs b/src/query/functions/tests/it/scalars/parser.rs index 68fdf0dab373..3f5d0cd7910b 100644 --- a/src/query/functions/tests/it/scalars/parser.rs +++ b/src/query/functions/tests/it/scalars/parser.rs @@ -149,8 +149,8 @@ pub fn transform_expr(ast: AExpr, columns: &[(&str, DataType)]) -> RawExpr { params: params .into_iter() .map(|param| match param { - ASTLiteral::UInt64(u) => u as usize, - ASTLiteral::Decimal256 { .. } => 0_usize, + ASTLiteral::UInt64(u) => Scalar::Number((u as i64).into()), + ASTLiteral::Decimal256 { .. } => Scalar::Number(0i64.into()), _ => unimplemented!(), }) .collect(), @@ -349,9 +349,13 @@ pub fn transform_expr(ast: AExpr, columns: &[(&str, DataType)]) -> RawExpr { }, ]), MapAccessor::DotNumber { key } => { - (vec![key as usize], vec![transform_expr(*expr, columns)]) + (vec![key as i64], vec![transform_expr(*expr, columns)]) } }; + let params = params + .into_iter() + .map(|x| Scalar::Number(x.into())) + .collect(); RawExpr::FunctionCall { span, name: "get".to_string(), diff --git a/src/query/functions/tests/it/scalars/testdata/math.txt b/src/query/functions/tests/it/scalars/testdata/math.txt index d959a713f93c..90b313cefd10 100644 --- a/src/query/functions/tests/it/scalars/testdata/math.txt +++ b/src/query/functions/tests/it/scalars/testdata/math.txt @@ -281,7 +281,7 @@ evaluation (internal): ast : round(-1.23) raw expr : round(minus(1.23)) -checked expr : round(76)(minus(1.23_d128(3,2))) +checked expr : round(0)(minus(1.23_d128(3,2))) optimized expr : -1_d128(3,0) output type : Decimal(3, 0) output domain : {-1..=-1} @@ -290,7 +290,7 @@ output : -1 ast : round(1.298, 1) raw expr : round(1.298, 1) -checked expr : round(77)(1.298_d128(4,3), 1_u8) +checked expr : round(1)(1.298_d128(4,3), 1_u8) optimized expr : 1.3_d128(4,1) output type : Decimal(4, 1) output domain : {1.3..=1.3} @@ -299,7 +299,7 @@ output : 1.3 ast : round(1.298, 0) raw expr : round(1.298, 0) -checked expr : round(76)(1.298_d128(4,3), 0_u8) +checked expr : round(0)(1.298_d128(4,3), 0_u8) optimized expr : 1_d128(4,0) output type : Decimal(4, 0) output domain : {1..=1} @@ -308,7 +308,7 @@ output : 1 ast : round(23.298, -1) raw expr : round(23.298, minus(1)) -checked expr : round(75)(23.298_d128(5,3), minus(1_u8)) +checked expr : round(-1)(23.298_d128(5,3), minus(1_u8)) optimized expr : 20_d128(5,0) output type : Decimal(5, 0) output domain : {20..=20} @@ -317,7 +317,7 @@ output : 20 ast : round(0.12345678901234567890123456789012345, 35) raw expr : round(0.12345678901234567890123456789012345, 35) -checked expr : round(111)(0.12345678901234567890123456789012345_d128(35,35), 35_u8) +checked expr : round(35)(0.12345678901234567890123456789012345_d128(35,35), 35_u8) optimized expr : 0.12345678901234567890123456789012345_d128(35,35) output type : Decimal(35, 35) output domain : {0.12345678901234567890123456789012345..=0.12345678901234567890123456789012345} @@ -410,7 +410,7 @@ evaluation (internal): ast : truncate(1.223, 1) raw expr : truncate(1.223, 1) -checked expr : truncate(77)(1.223_d128(4,3), 1_u8) +checked expr : truncate(1)(1.223_d128(4,3), 1_u8) optimized expr : 1.2_d128(4,1) output type : Decimal(4, 1) output domain : {1.2..=1.2} @@ -419,7 +419,7 @@ output : 1.2 ast : truncate(1.999) raw expr : truncate(1.999) -checked expr : truncate(76)(1.999_d128(4,3)) +checked expr : truncate(0)(1.999_d128(4,3)) optimized expr : 1_d128(4,0) output type : Decimal(4, 0) output domain : {1..=1} @@ -428,7 +428,7 @@ output : 1 ast : truncate(1.999, 1) raw expr : truncate(1.999, 1) -checked expr : truncate(77)(1.999_d128(4,3), 1_u8) +checked expr : truncate(1)(1.999_d128(4,3), 1_u8) optimized expr : 1.9_d128(4,1) output type : Decimal(4, 1) output domain : {1.9..=1.9} @@ -446,7 +446,7 @@ output : 100 ast : truncate(10.28*100, 0) raw expr : truncate(multiply(10.28, 100), 0) -checked expr : truncate(76)(multiply(to_decimal(7, 2)(10.28_d128(4,2)), to_decimal(7, 0)(100_u8)), 0_u8) +checked expr : truncate(0)(multiply(to_decimal(7, 2)(10.28_d128(4,2)), to_decimal(7, 0)(100_u8)), 0_u8) optimized expr : 1028_d128(7,0) output type : Decimal(7, 0) output domain : {1028..=1028} diff --git a/src/query/sql/src/planner/binder/aggregate.rs b/src/query/sql/src/planner/binder/aggregate.rs index 4bb10ad3c696..d69587387ddc 100644 --- a/src/query/sql/src/planner/binder/aggregate.rs +++ b/src/query/sql/src/planner/binder/aggregate.rs @@ -282,7 +282,7 @@ impl<'a> AggregateRewriter<'a> { let mut replaced_params = Vec::with_capacity(function.arguments.len()); for arg in &function.arguments { if let Some(index) = agg_info.group_items_map.get(arg) { - replaced_params.push(*index); + replaced_params.push(*index as _); } else { return Err(ErrorCode::BadArguments( "Arguments of grouping should be group by expressions", diff --git a/src/query/sql/src/planner/binder/table.rs b/src/query/sql/src/planner/binder/table.rs index bc56357f61d1..6a3646c57879 100644 --- a/src/query/sql/src/planner/binder/table.rs +++ b/src/query/sql/src/planner/binder/table.rs @@ -378,7 +378,7 @@ impl Binder { let field_expr = ScalarExpr::FunctionCall(FunctionCall { span: *span, func_name: "get".to_string(), - params: vec![i + 1], + params: vec![(i + 1) as i64], arguments: vec![scalar.clone()], }); let data_type = field_expr.data_type()?; diff --git a/src/query/sql/src/planner/optimizer/rule/utils/constant.rs b/src/query/sql/src/planner/optimizer/rule/utils/constant.rs index 2fdad26bac06..16d123e04b8b 100644 --- a/src/query/sql/src/planner/optimizer/rule/utils/constant.rs +++ b/src/query/sql/src/planner/optimizer/rule/utils/constant.rs @@ -204,7 +204,7 @@ pub fn remove_trivial_type_cast(left: ScalarExpr, right: ScalarExpr) -> (ScalarE (**argument).clone(), ScalarExpr::ConstantExpr(ConstantExpr { span: *span, - value: Scalar::Number(NumberScalar::Int64(v)), + value: Scalar::Number(v.into()), }), ); } diff --git a/src/query/sql/src/planner/plans/scalar_expr.rs b/src/query/sql/src/planner/plans/scalar_expr.rs index 3c7ed84010c6..d8e05ea9c6c5 100644 --- a/src/query/sql/src/planner/plans/scalar_expr.rs +++ b/src/query/sql/src/planner/plans/scalar_expr.rs @@ -526,7 +526,7 @@ pub struct FunctionCall { #[educe(Hash(ignore), PartialEq(ignore), Eq(ignore))] pub span: Span, pub func_name: String, - pub params: Vec, + pub params: Vec, pub arguments: Vec, } diff --git a/src/query/sql/src/planner/semantic/lowering.rs b/src/query/sql/src/planner/semantic/lowering.rs index dfaad7b13512..22c9162dea10 100644 --- a/src/query/sql/src/planner/semantic/lowering.rs +++ b/src/query/sql/src/planner/semantic/lowering.rs @@ -22,6 +22,7 @@ use databend_common_expression::ColumnIndex; use databend_common_expression::DataSchema; use databend_common_expression::Expr; use databend_common_expression::RawExpr; +use databend_common_expression::Scalar; use databend_common_functions::BUILTIN_FUNCTIONS; use crate::binder::ColumnBindingBuilder; @@ -229,7 +230,11 @@ impl ScalarExpr { ScalarExpr::FunctionCall(func) => RawExpr::FunctionCall { span: func.span, name: func.func_name.clone(), - params: func.params.clone(), + params: func + .params + .iter() + .map(|x| Scalar::Number((*x).into())) + .collect(), args: func.arguments.iter().map(ScalarExpr::as_raw_expr).collect(), }, ScalarExpr::CastExpr(cast) => RawExpr::Cast { diff --git a/src/query/sql/src/planner/semantic/type_check.rs b/src/query/sql/src/planner/semantic/type_check.rs index e47550676a06..7b55453bff26 100644 --- a/src/query/sql/src/planner/semantic/type_check.rs +++ b/src/query/sql/src/planner/semantic/type_check.rs @@ -861,7 +861,7 @@ impl<'a> TypeChecker<'a> { let params = params .iter() .map(|literal| match literal { - Literal::UInt64(n) => Ok(*n as usize), + Literal::UInt64(n) => Ok(*n as i64), lit => Err(ErrorCode::SemanticError(format!( "invalid parameter {lit} for scalar function" )) @@ -1054,7 +1054,7 @@ impl<'a> TypeChecker<'a> { if let databend_common_expression::Scalar::Number(NumberScalar::UInt8(0)) = expr.value { args[1] = ConstantExpr { span: expr.span, - value: databend_common_expression::Scalar::Number(NumberScalar::Int64(1)), + value: databend_common_expression::Scalar::Number(1i64.into()), } .into(); } @@ -1819,7 +1819,7 @@ impl<'a> TypeChecker<'a> { &mut self, span: Span, func_name: &str, - params: Vec, + params: Vec, arguments: &[&Expr], ) -> Result> { // Check if current function is a virtual function, e.g. `database`, `version` @@ -1885,7 +1885,7 @@ impl<'a> TypeChecker<'a> { &self, span: Span, func_name: &str, - params: Vec, + params: Vec, args: Vec, ) -> Result> { // Type check @@ -1893,7 +1893,7 @@ impl<'a> TypeChecker<'a> { let raw_expr = RawExpr::FunctionCall { span, name: func_name.to_string(), - params: params.clone(), + params: params.iter().map(|x| Scalar::Number((*x).into())).collect(), args: arguments, }; let expr = type_check::check(&raw_expr, &BUILTIN_FUNCTIONS)?; @@ -2947,7 +2947,7 @@ impl<'a> TypeChecker<'a> { let value = FunctionCall { span, - params: vec![idx + 1], + params: vec![(idx + 1) as _], arguments: vec![scalar.clone()], func_name: "get".to_string(), } @@ -3073,7 +3073,7 @@ impl<'a> TypeChecker<'a> { scalar = FunctionCall { span: expr.span(), func_name: "get".to_string(), - params: vec![idx], + params: vec![idx as _], arguments: vec![scalar.clone()], } .into(); @@ -3189,7 +3189,7 @@ impl<'a> TypeChecker<'a> { while let Some((idx, table_data_type)) = index_with_types.pop_front() { scalar = FunctionCall { span, - params: vec![idx], + params: vec![idx as _], arguments: vec![scalar.clone()], func_name: "get".to_string(), } From 37e04b9922ead3f87d7038518e4153f4284b5247 Mon Sep 17 00:00:00 2001 From: BohuTANG Date: Tue, 19 Dec 2023 19:46:41 +0800 Subject: [PATCH 19/20] chore: fix typo and refine the merge into status (#14081) --- .../interpreters/interpreter_merge_into.rs | 2 +- src/query/sql/src/planner/mod.rs | 2 +- src/query/sql/src/planner/plans/merge_into.rs | 66 +++++++++++-------- src/query/sql/src/planner/plans/mod.rs | 2 +- .../system/src/virtual_columns_table.rs | 12 ++-- 5 files changed, 46 insertions(+), 38 deletions(-) diff --git a/src/query/service/src/interpreters/interpreter_merge_into.rs b/src/query/service/src/interpreters/interpreter_merge_into.rs index aae4a1b114a9..87d9ea9c0219 100644 --- a/src/query/service/src/interpreters/interpreter_merge_into.rs +++ b/src/query/service/src/interpreters/interpreter_merge_into.rs @@ -477,7 +477,7 @@ impl MergeIntoInterpreter { plans::INSERT_NAME => { columns.push(UInt32Type::from_data(vec![status.insert_rows as u32])) } - plans::UPDTAE_NAME => { + plans::UPDATE_NAME => { columns.push(UInt32Type::from_data(vec![status.update_rows as u32])) } plans::DELETE_NAME => { diff --git a/src/query/sql/src/planner/mod.rs b/src/query/sql/src/planner/mod.rs index 13f5162b67d0..1e6ff878462f 100644 --- a/src/query/sql/src/planner/mod.rs +++ b/src/query/sql/src/planner/mod.rs @@ -45,6 +45,6 @@ pub use plans::insert::InsertInputSource; pub use plans::ScalarExpr; pub use plans::DELETE_NAME; pub use plans::INSERT_NAME; -pub use plans::UPDTAE_NAME; +pub use plans::UPDATE_NAME; pub use semantic::*; pub use stream_column::*; diff --git a/src/query/sql/src/planner/plans/merge_into.rs b/src/query/sql/src/planner/plans/merge_into.rs index a2251eda6268..37ffa1276f4d 100644 --- a/src/query/sql/src/planner/plans/merge_into.rs +++ b/src/query/sql/src/planner/plans/merge_into.rs @@ -78,14 +78,14 @@ impl std::fmt::Debug for MergeInto { .field("table_id", &self.table_id) .field("join", &self.input) .field("matched", &self.matched_evaluators) - .field("unmateched", &self.unmatched_evaluators) + .field("unmatched", &self.unmatched_evaluators) .field("distributed", &self.distributed) .finish() } } -pub const INSERT_NAME: &str = "number of rows insertd"; -pub const UPDTAE_NAME: &str = "number of rows updated"; +pub const INSERT_NAME: &str = "number of rows inserted"; +pub const UPDATE_NAME: &str = "number of rows updated"; pub const DELETE_NAME: &str = "number of rows deleted"; impl MergeInto { @@ -107,33 +107,41 @@ impl MergeInto { } fn merge_into_table_schema(&self) -> Result { - let field_insertd = DataField::new(INSERT_NAME, DataType::Number(NumberDataType::Int32)); - let field_updated = DataField::new(UPDTAE_NAME, DataType::Number(NumberDataType::Int32)); - let field_deleted = DataField::new(DELETE_NAME, DataType::Number(NumberDataType::Int32)); - match self.merge_into_mutations() { - (true, true, true) => Ok(DataSchemaRefExt::create(vec![ - field_insertd.clone(), - field_updated.clone(), - field_deleted.clone(), - ])), - (true, true, false) => Ok(DataSchemaRefExt::create(vec![ - field_insertd.clone(), - field_updated.clone(), - ])), - (true, false, true) => Ok(DataSchemaRefExt::create(vec![ - field_insertd.clone(), - field_deleted.clone(), - ])), - (true, false, false) => Ok(DataSchemaRefExt::create(vec![field_insertd.clone()])), - (false, true, true) => Ok(DataSchemaRefExt::create(vec![ - field_updated.clone(), - field_deleted.clone(), - ])), - (false, true, false) => Ok(DataSchemaRefExt::create(vec![field_updated.clone()])), - (false, false, true) => Ok(DataSchemaRefExt::create(vec![field_deleted.clone()])), - _ => Err(ErrorCode::BadArguments( + let (insert, update, delete) = self.merge_into_mutations(); + + let fields = [ + ( + DataField::new(INSERT_NAME, DataType::Number(NumberDataType::Int32)), + insert, + ), + ( + DataField::new(UPDATE_NAME, DataType::Number(NumberDataType::Int32)), + update, + ), + ( + DataField::new(DELETE_NAME, DataType::Number(NumberDataType::Int32)), + delete, + ), + ]; + + // Filter and collect the fields to include in the schema. + // Only fields with a corresponding true value in the mutation states are included. + let schema_fields: Vec = fields + .iter() + .filter_map( + |(field, include)| { + if *include { Some(field.clone()) } else { None } + }, + ) + .collect(); + + // Check if any fields are included. If none, return an error. Otherwise, return the schema. + if schema_fields.is_empty() { + Err(ErrorCode::BadArguments( "at least one matched or unmatched clause for merge into", - )), + )) + } else { + Ok(DataSchemaRefExt::create(schema_fields)) } } diff --git a/src/query/sql/src/planner/plans/mod.rs b/src/query/sql/src/planner/plans/mod.rs index b971477e0cd4..04e5dff0da5f 100644 --- a/src/query/sql/src/planner/plans/mod.rs +++ b/src/query/sql/src/planner/plans/mod.rs @@ -77,7 +77,7 @@ pub use merge_into::MergeInto; pub use merge_into::UnmatchedEvaluator; pub use merge_into::DELETE_NAME; pub use merge_into::INSERT_NAME; -pub use merge_into::UPDTAE_NAME; +pub use merge_into::UPDATE_NAME; pub use operator::*; pub use pattern::PatternPlan; pub use plan::*; diff --git a/src/query/storages/system/src/virtual_columns_table.rs b/src/query/storages/system/src/virtual_columns_table.rs index 1dbf511d5d31..670226f8722a 100644 --- a/src/query/storages/system/src/virtual_columns_table.rs +++ b/src/query/storages/system/src/virtual_columns_table.rs @@ -67,8 +67,8 @@ impl AsyncSystemTable for VirtualColumnsTable { let mut database_names = Vec::with_capacity(virtual_column_metas.len()); let mut table_names = Vec::with_capacity(virtual_column_metas.len()); let mut virtual_columns = Vec::with_capacity(virtual_column_metas.len()); - let mut created_ons = Vec::with_capacity(virtual_column_metas.len()); - let mut updated_ons = Vec::with_capacity(virtual_column_metas.len()); + let mut created_on_columns = Vec::with_capacity(virtual_column_metas.len()); + let mut updated_on_columns = Vec::with_capacity(virtual_column_metas.len()); if !virtual_column_metas.is_empty() { let mut virtual_column_meta_map: HashMap = virtual_column_metas @@ -90,8 +90,8 @@ impl AsyncSystemTable for VirtualColumnsTable { .as_bytes() .to_vec(), ); - created_ons.push(virtual_column_meta.created_on.timestamp_micros()); - updated_ons + created_on_columns.push(virtual_column_meta.created_on.timestamp_micros()); + updated_on_columns .push(virtual_column_meta.updated_on.map(|u| u.timestamp_micros())); } } @@ -102,8 +102,8 @@ impl AsyncSystemTable for VirtualColumnsTable { StringType::from_data(database_names), StringType::from_data(table_names), StringType::from_data(virtual_columns), - TimestampType::from_data(created_ons), - TimestampType::from_opt_data(updated_ons), + TimestampType::from_data(created_on_columns), + TimestampType::from_opt_data(updated_on_columns), ])) } } From c76b37e0fb920e167478be1f1ac20032443fa1da Mon Sep 17 00:00:00 2001 From: Liuqing Yue Date: Wed, 20 Dec 2023 10:31:23 +0800 Subject: [PATCH 20/20] docs: replace the wrong links in README (#14090) fix: replace the wrong links in README Signed-off-by: Liuqing Yue --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index de55943faf87..c9550bd69608 100644 --- a/README.md +++ b/README.md @@ -226,8 +226,8 @@ Databend thrives on community contributions! Whether it's through ideas, code, o Here are some resources to help you get started: -- [Building Databend From Source](https://docs.databend.com/doc/contributing/building-from-source) -- [The First Good Pull Request](https://docs.databend.com/doc/contributing/good-pr) +- [Building Databend From Source](https://docs.databend.com/doc/overview/community/contributor/building-from-source) +- [The First Good Pull Request](https://docs.databend.com/doc/overview/community/contributor/good-pr) ## 👥 Community