Merge branch 'main' into add-warnings-to-query-ctx

.github/actions/build_macos/action.yml

@@ -21,6 +21,7 @@ runs:
     - name: Rust setup
       shell: bash
       run: |
+        brew unlink python || true
         bash ./scripts/setup/dev_setup.sh -yb
         rustup target add ${{ inputs.target }}
 

.github/actions/test_sqllogic_stage/action.yml

@@ -28,7 +28,6 @@ runs:
         target: ${{ inputs.target }}
         artifacts: sqllogictests,meta,query
     - name: Minio Setup for (ubuntu-latest only)
-      if: inputs.storage == 's3'
       shell: bash
       run: |
         docker run -d --network host --name minio \

Cargo.toml

@@ -119,7 +119,7 @@ opendal = { version = "0.42", features = [
 ] }
 ethnum = { version = "1.3.2" }
 ordered-float = { version = "3.6.0", default-features = false }
-jsonb = { git = "https://github.com/datafuselabs/jsonb", rev = "1d7a3e9" }
+jsonb = { git = "https://github.com/datafuselabs/jsonb", rev = "582c139" }
 
 # openraft = { version = "0.8.2", features = ["compat-07"] }
 # For debugging

clippy.toml

@@ -0,0 +1,16 @@
+disallowed-methods = [
+    { path = "std::panic::catch_unwind", reason = "Please use `common_base::runtime::catch_unwind` instead." },
+    { path = "futures::FutureExt::catch_unwind", reason = "Please use `common_base::runtime::CatchUnwindFuture` instead." },
+    { path = "num_traits::sign::Signed::is_positive", reason = "This returns true for 0.0 but false for 0." },
+    { path = "num_traits::sign::Signed::is_negative", reason = "This returns true for -0.0 but false for 0." },
+    { path = "num_traits::sign::Signed::signum", reason = "This returns 1.0 for 0.0 but 0 for 0." }
+]
+
+## TODO: enable it in next pr
+# disallowed-macros = [
+#     { path = "lazy_static::lazy_static", reason = "Please use `std::sync::LazyLock` instead." },
+# ]
+
+avoid-breaking-exported-api = true
+too-many-arguments-threshold = 10
+upper-case-acronyms-aggressive = false

scripts/setup/dev_setup.sh

@@ -98,6 +98,35 @@ function install_build_essentials {
 	esac
 }
 
+function install_ziglang {
+	PACKAGE_MANAGER=$1
+
+	if zig version; then
+		echo "==> ziglang is already installed"
+		return
+	fi
+	echo "==> installing ziglang..."
+
+	arch=$(uname -m)
+	case "$PACKAGE_MANAGER" in
+	apt-get | yum | dnf | pacman)
+		curl -sSfLo /tmp/zig.tar.xz "https://ziglang.org/download/0.11.0/zig-linux-${arch}-0.11.0.tar.xz"
+		tar -xf /tmp/zig.tar.xz -C /tmp
+		"${PRE_COMMAND[@]}" mv "/tmp/zig-linux-${arch}-0.11.0/zig" /usr/local/bin/
+		"${PRE_COMMAND[@]}" chmod +x /usr/local/bin/zig
+		"${PRE_COMMAND[@]}" mv "/tmp/zig-linux-${arch}-0.11.0/lib" /usr/local/lib/zig
+		rm -rf /tmp/zig*
+		;;
+	brew)
+		install_pkg zig "$PACKAGE_MANAGER"
+		;;
+	*)
+		echo "Unable to install ziglang with package manager: $PACKAGE_MANAGER"
+		exit 1
+		;;
+	esac
+}
+
 function install_python3 {
 	PACKAGE_MANAGER=$1
 
@@ -525,7 +554,7 @@ if [[ "$INSTALL_BUILD_TOOLS" == "true" ]]; then
 	install_pkg cmake "$PACKAGE_MANAGER"
 	install_pkg clang "$PACKAGE_MANAGER"
 	install_pkg llvm "$PACKAGE_MANAGER"
-	install_pkg zig "$PACKAGE_MANAGER"
+	install_ziglang "$PACKAGE_MANAGER"
 	install_python3 "$PACKAGE_MANAGER"
 
 	# Any call to cargo will make rustup install the correct toolchain

src/common/arrow/src/arrow/array/dictionary/typed_iterator.rs

@@ -47,9 +47,9 @@ impl<O: Offset> DictValue for Utf8Array<O> {
         array
             .as_any()
             .downcast_ref::<Self>()
-            .ok_or(Error::InvalidArgumentError(
-                "could not convert array to dictionary value".into(),
-            ))
+            .ok_or_else(|| {
+                Error::InvalidArgumentError("could not convert array to dictionary value".into())
+            })
             .map(|arr| {
                 assert_eq!(
                     arr.null_count(),

src/common/arrow/src/native/compression/basic.rs

@@ -47,7 +47,7 @@ impl TryFrom<&Compression> for CommonCompression {
 }
 
 impl CommonCompression {
-    pub fn to_compression(&self) -> Compression {
+    pub fn to_compression(self) -> Compression {
         match self {
             Self::None => Compression::None,
             Self::Lz4 => Compression::Lz4,

src/common/base/src/base/take_mut.rs

@@ -16,6 +16,8 @@ use std::panic;
 
 use common_exception::Result;
 
+use crate::runtime::catch_unwind;
+
 /// copy from https://docs.rs/take_mut/0.2.2/take_mut/fn.take.html with some modifications.
 /// if a panic occurs, the entire process will be aborted, as there's no valid `T` to put back into the `&mut T`.
 pub fn take_mut<T, F>(mut_ref: &mut T, closure: F) -> Result<()>
@@ -24,7 +26,7 @@ where F: FnOnce(T) -> Result<T> {
 
     unsafe {
         let old_t = ptr::read(mut_ref);
-        let closure_result = panic::catch_unwind(panic::AssertUnwindSafe(|| closure(old_t)));
+        let closure_result = catch_unwind(panic::AssertUnwindSafe(|| closure(old_t)));
 
         match closure_result {
             Ok(Ok(new_t)) => {

src/common/base/src/lib.rs

@@ -23,6 +23,7 @@
 #![feature(backtrace_frames)]
 #![feature(alloc_error_hook)]
 #![feature(slice_swap_unchecked)]
+#![feature(lint_reasons)]
 
 pub mod base;
 pub mod containers;

src/common/base/src/runtime/catch_unwind.rs

@@ -23,6 +23,7 @@ use futures::future::BoxFuture;
 use futures::FutureExt;
 
 pub fn catch_unwind<F: FnOnce() -> R, R>(f: F) -> Result<R> {
+    #[expect(clippy::disallowed_methods)]
     match std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)) {
         Ok(res) => Ok(res),
         Err(cause) => match cause.downcast_ref::<&'static str>() {

src/common/storage/src/lib.rs

@@ -65,10 +65,12 @@ pub use stage::StageFilesInfo;
 pub use stage::STDIN_FD;
 
 mod copy;
+mod merge;
 mod statistics;
 
 pub use copy::CopyStatus;
 pub use copy::FileParseError;
 pub use copy::FileStatus;
+pub use merge::MergeStatus;
 pub use statistics::Datum;
 pub use statistics::F64;

src/common/storage/src/merge.rs

@@ -0,0 +1,43 @@
+// Copyright 2021 Datafuse Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use serde::Deserialize;
+use serde::Serialize;
+
+#[derive(Default, Clone, Serialize, Deserialize)]
+pub struct MergeStatus {
+    pub insert_rows: usize,
+    pub deleted_rows: usize,
+    pub update_rows: usize,
+}
+
+impl MergeStatus {
+    pub fn add_insert_rows(&mut self, insert_rows: usize) {
+        self.insert_rows += insert_rows;
+    }
+
+    pub fn add_deleted_rows(&mut self, deleted_rows: usize) {
+        self.deleted_rows += deleted_rows
+    }
+
+    pub fn add_update_rows(&mut self, update_rows: usize) {
+        self.update_rows += update_rows
+    }
+
+    pub fn merge_status(&mut self, merge_status: MergeStatus) {
+        self.insert_rows += merge_status.insert_rows;
+        self.deleted_rows += merge_status.deleted_rows;
+        self.update_rows += merge_status.update_rows;
+    }
+}

src/common/storage/src/stage.rs

@@ -151,12 +151,16 @@ impl StageFilesInfo {
     #[async_backtrace::framed]
     pub async fn first_file(&self, operator: &Operator) -> Result<StageFileInfo> {
         let mut files = self.list(operator, true, None).await?;
-        files.pop().ok_or(ErrorCode::BadArguments("no file found"))
+        files
+            .pop()
+            .ok_or_else(|| ErrorCode::BadArguments("no file found"))
     }
 
     pub fn blocking_first_file(&self, operator: &Operator) -> Result<StageFileInfo> {
         let mut files = self.blocking_list(operator, true, None)?;
-        files.pop().ok_or(ErrorCode::BadArguments("no file found"))
+        files
+            .pop()
+            .ok_or_else(|| ErrorCode::BadArguments("no file found"))
     }
 
     pub fn blocking_list(

src/meta/api/src/schema_api_impl.rs

@@ -552,18 +552,19 @@ impl<KV: kvapi::KVApi<Error = MetaError> + ?Sized> SchemaApi for KV {
                     UndropDbHasNoHistory::new(&name_key.db_name),
                 )));
             } else {
-                db_id_list_opt.ok_or(KVAppError::AppError(AppError::UndropDbHasNoHistory(
-                    UndropDbHasNoHistory::new(&name_key.db_name),
-                )))?
+                db_id_list_opt.ok_or_else(|| {
+                    KVAppError::AppError(AppError::UndropDbHasNoHistory(UndropDbHasNoHistory::new(
+                        &name_key.db_name,
+                    )))
+                })?
             };
 
             // Return error if there is no db id history.
-            let db_id =
-                *db_id_list
-                    .last()
-                    .ok_or(KVAppError::AppError(AppError::UndropDbHasNoHistory(
-                        UndropDbHasNoHistory::new(&name_key.db_name),
-                    )))?;
+            let db_id = *db_id_list.last().ok_or_else(|| {
+                KVAppError::AppError(AppError::UndropDbHasNoHistory(UndropDbHasNoHistory::new(
+                    &name_key.db_name,
+                )))
+            })?;
 
             // get db_meta of the last db id
             let dbid = DatabaseId { db_id };
@@ -1745,9 +1746,11 @@ impl<KV: kvapi::KVApi<Error = MetaError> + ?Sized> SchemaApi for KV {
                     UndropTableHasNoHistory::new(&tenant_dbname_tbname.table_name),
                 )));
             } else {
-                tb_id_list_opt.ok_or(KVAppError::AppError(AppError::UndropTableHasNoHistory(
-                    UndropTableHasNoHistory::new(&tenant_dbname_tbname.table_name),
-                )))?
+                tb_id_list_opt.ok_or_else(|| {
+                    KVAppError::AppError(AppError::UndropTableHasNoHistory(
+                        UndropTableHasNoHistory::new(&tenant_dbname_tbname.table_name),
+                    ))
+                })?
             };
 
             // Return error if there is no table id history.
@@ -2339,10 +2342,12 @@ impl<KV: kvapi::KVApi<Error = MetaError> + ?Sized> SchemaApi for KV {
             let (_, table_name_opt): (_, Option<DBIdTableName>) =
                 get_pb_value(self, &table_id_to_name).await?;
 
-            let dbid_tbname =
-                table_name_opt.ok_or(KVAppError::AppError(AppError::UnknownTableId(
-                    UnknownTableId::new(table_id, "drop_table_by_id failed to find db_id"),
-                )))?;
+            let dbid_tbname = table_name_opt.ok_or_else(|| {
+                KVAppError::AppError(AppError::UnknownTableId(UnknownTableId::new(
+                    table_id,
+                    "drop_table_by_id failed to find db_id",
+                )))
+            })?;
 
             let db_id = dbid_tbname.db_id;
             let tbname = dbid_tbname.table_name.clone();