From 400365b8c5a9079f673f0ace3dd7e48cc35ebe65 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Thu, 14 Dec 2023 16:27:05 +0800 Subject: [PATCH] feat: add column create_at for stages table --- scripts/build/build-debug.sh | 8 ++++++++ src/meta/app/src/principal/user_stage.rs | 1 + src/meta/proto-conv/src/stage_from_to_protobuf_impl.rs | 5 +++++ src/meta/proto-conv/tests/it/user_proto_conv.rs | 1 + src/meta/proto-conv/tests/it/v025_user_stage.rs | 3 +++ src/meta/proto-conv/tests/it/v035_user_stage.rs | 3 +++ .../proto-conv/tests/it/v042_s3_stage_new_field.rs | 3 +++ src/meta/protos/proto/stage.proto | 3 ++- .../src/interpreters/interpreter_user_stage_create.rs | 2 ++ .../tests/it/storages/testdata/columns_table.txt | 1 + src/query/sql/src/planner/binder/binder.rs | 2 +- src/query/storages/system/src/stages_table.rs | 5 +++++ .../suites/base/05_ddl/05_0016_ddl_stage.test | 10 ++-------- .../suites/base/06_show/06_0011_show_stages.test | 9 ++------- 14 files changed, 39 insertions(+), 17 deletions(-) diff --git a/scripts/build/build-debug.sh b/scripts/build/build-debug.sh index 373c85e971d9..d1d8eff684e3 100755 --- a/scripts/build/build-debug.sh +++ b/scripts/build/build-debug.sh @@ -8,5 +8,13 @@ SCRIPT_PATH="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)" cd "$SCRIPT_PATH/../.." || exit echo "Build(DEBUG) start..." + +if [[ $(uname -a) =~ Darwin ]]; then + export CMAKE_CC_COMPILER=/opt/homebrew/opt/llvm/bin/clang + export CMAKE_CXX_COMPILER=/opt/homebrew/opt/llvm/bin/clang++ + export JEMALLOC_SYS_WITH_LG_PAGE=14 + export JEMALLOC_SYS_WITH_MALLOC_CONF=oversize_threshold:0,dirty_decay_ms:5000,muzzy_decay_ms:5000 +fi + cargo build --bin=databend-query --bin=databend-meta --bin=databend-metactl --bin=databend-sqllogictests --bin=databend-sqlsmith echo "All done..." diff --git a/src/meta/app/src/principal/user_stage.rs b/src/meta/app/src/principal/user_stage.rs index 2077fbfff960..9382ab89965b 100644 --- a/src/meta/app/src/principal/user_stage.rs +++ b/src/meta/app/src/principal/user_stage.rs @@ -569,6 +569,7 @@ pub struct StageInfo { /// TODO(xuanwo): stage doesn't have this info anymore, remove it. pub number_of_files: u64, pub creator: Option, + pub created_on: DateTime, } impl StageInfo { diff --git a/src/meta/proto-conv/src/stage_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/stage_from_to_protobuf_impl.rs index 40c5a14b3272..b8bb83693f03 100644 --- a/src/meta/proto-conv/src/stage_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/stage_from_to_protobuf_impl.rs @@ -220,6 +220,10 @@ impl FromToProto for mt::principal::StageInfo { Some(c) => Some(mt::principal::UserIdentity::from_pb(c)?), None => None, }, + created_on: match p.created_on { + Some(c) => DateTime::::from_pb(c)?, + None => DateTime::::default(), + }, }) } @@ -241,6 +245,7 @@ impl FromToProto for mt::principal::StageInfo { Some(c) => Some(mt::principal::UserIdentity::to_pb(c)?), None => None, }, + created_on: Some(self.created_on.to_pb()?), }) } } diff --git a/src/meta/proto-conv/tests/it/user_proto_conv.rs b/src/meta/proto-conv/tests/it/user_proto_conv.rs index b8cc3327620b..9e7be26afc61 100644 --- a/src/meta/proto-conv/tests/it/user_proto_conv.rs +++ b/src/meta/proto-conv/tests/it/user_proto_conv.rs @@ -103,6 +103,7 @@ pub(crate) fn test_fs_stage_info() -> mt::principal::StageInfo { username: "databend".to_string(), hostname: "databend.rs".to_string(), }), + created_on: Utc::now(), } } diff --git a/src/meta/proto-conv/tests/it/v025_user_stage.rs b/src/meta/proto-conv/tests/it/v025_user_stage.rs index 82add6d8f7e8..9a751dd82dd6 100644 --- a/src/meta/proto-conv/tests/it/v025_user_stage.rs +++ b/src/meta/proto-conv/tests/it/v025_user_stage.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use chrono::DateTime; +use chrono::Utc; use common_meta_app as mt; use common_meta_app::principal::UserIdentity; use common_meta_app::storage::StorageFsConfig; @@ -73,6 +75,7 @@ fn test_decode_v25_user_stage() -> anyhow::Result<()> { username: "databend".to_string(), hostname: "databend.rs".to_string(), }), + created_on: DateTime::::default(), }; common::test_load_old(func_name!(), stage_info_v25.as_slice(), 25, want())?; common::test_pb_from_to(func_name!(), want())?; diff --git a/src/meta/proto-conv/tests/it/v035_user_stage.rs b/src/meta/proto-conv/tests/it/v035_user_stage.rs index e4478e16d86c..daddf737a9e2 100644 --- a/src/meta/proto-conv/tests/it/v035_user_stage.rs +++ b/src/meta/proto-conv/tests/it/v035_user_stage.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use chrono::DateTime; +use chrono::Utc; use common_meta_app as mt; use common_meta_app::principal::UserIdentity; use common_meta_app::storage::StorageFsConfig; @@ -70,6 +72,7 @@ fn test_decode_v35_user_stage() -> anyhow::Result<()> { username: "databend".to_string(), hostname: "databend.rs".to_string(), }), + created_on: DateTime::::default(), }; common::test_load_old(func_name!(), stage_info_v35.as_slice(), 35, want())?; common::test_pb_from_to(func_name!(), want())?; diff --git a/src/meta/proto-conv/tests/it/v042_s3_stage_new_field.rs b/src/meta/proto-conv/tests/it/v042_s3_stage_new_field.rs index f888a9928dfc..fa6087dd24cf 100644 --- a/src/meta/proto-conv/tests/it/v042_s3_stage_new_field.rs +++ b/src/meta/proto-conv/tests/it/v042_s3_stage_new_field.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use chrono::DateTime; +use chrono::Utc; use common_meta_app as mt; use common_meta_app::principal::UserIdentity; use common_meta_app::storage::StorageParams; @@ -74,6 +76,7 @@ fn test_decode_v42_s3_stage_new_field() -> anyhow::Result<()> { username: "databend".to_string(), hostname: "databend.rs".to_string(), }), + created_on: DateTime::::default(), }; common::test_load_old(func_name!(), stage_info_v42.as_slice(), 42, want())?; diff --git a/src/meta/protos/proto/stage.proto b/src/meta/protos/proto/stage.proto index 5095229078c6..b46016893ebc 100644 --- a/src/meta/protos/proto/stage.proto +++ b/src/meta/protos/proto/stage.proto @@ -74,6 +74,8 @@ message StageInfo { optional UserIdentity creator = 8; FileFormatParams file_format_params = 9; + + optional string created_on = 10; } message StageFile { @@ -87,4 +89,3 @@ message StageFile { optional UserIdentity creator = 5; optional string etag = 6; } - diff --git a/src/query/service/src/interpreters/interpreter_user_stage_create.rs b/src/query/service/src/interpreters/interpreter_user_stage_create.rs index d95c98284749..53bafbca7cdf 100644 --- a/src/query/service/src/interpreters/interpreter_user_stage_create.rs +++ b/src/query/service/src/interpreters/interpreter_user_stage_create.rs @@ -14,6 +14,7 @@ use std::sync::Arc; +use chrono::Utc; use common_exception::ErrorCode; use common_exception::Result; use common_meta_app::principal::StageType; @@ -78,6 +79,7 @@ impl Interpreter for CreateUserStageInterpreter { let mut user_stage = user_stage; user_stage.creator = Some(self.ctx.get_current_user()?.identity()); + user_stage.created_on = Utc::now(); let _create_stage = user_mgr .add_stage(&plan.tenant, user_stage, plan.if_not_exists) .await?; diff --git a/src/query/service/tests/it/storages/testdata/columns_table.txt b/src/query/service/tests/it/storages/testdata/columns_table.txt index d40948af536a..893369478c61 100644 --- a/src/query/service/tests/it/storages/testdata/columns_table.txt +++ b/src/query/service/tests/it/storages/testdata/columns_table.txt @@ -66,6 +66,7 @@ DB.Table: 'system'.'columns', Table: columns-table_id:1, ver:0, Engine: SystemCo | 'created_on' | 'system' | 'background_tasks' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | | 'created_on' | 'system' | 'indexes' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | | 'created_on' | 'system' | 'locks' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | +| 'created_on' | 'system' | 'stages' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | | 'created_on' | 'system' | 'streams' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | | 'created_on' | 'system' | 'tables' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | | 'created_on' | 'system' | 'tables_with_history' | 'Timestamp' | 'TIMESTAMP' | '' | '' | 'NO' | '' | diff --git a/src/query/sql/src/planner/binder/binder.rs b/src/query/sql/src/planner/binder/binder.rs index a5eb0f205eff..6442e6feb970 100644 --- a/src/query/sql/src/planner/binder/binder.rs +++ b/src/query/sql/src/planner/binder/binder.rs @@ -349,7 +349,7 @@ impl<'a> Binder { })), // Stages - Statement::ShowStages => self.bind_rewrite_to_query(bind_context, "SELECT name, stage_type, number_of_files, creator, comment FROM system.stages ORDER BY name", RewriteKind::ShowStages).await?, + Statement::ShowStages => self.bind_rewrite_to_query(bind_context, "SELECT name, stage_type, number_of_files, creator, created_on, comment FROM system.stages ORDER BY name", RewriteKind::ShowStages).await?, Statement::ListStage { location, pattern } => { let pattern = if let Some(pattern) = pattern { format!(", pattern => '{pattern}'") diff --git a/src/query/storages/system/src/stages_table.rs b/src/query/storages/system/src/stages_table.rs index ae995b592a7e..9e83096d0b28 100644 --- a/src/query/storages/system/src/stages_table.rs +++ b/src/query/storages/system/src/stages_table.rs @@ -22,6 +22,7 @@ use common_exception::Result; use common_expression::types::number::UInt64Type; use common_expression::types::NumberDataType; use common_expression::types::StringType; +use common_expression::types::TimestampType; use common_expression::utils::FromData; use common_expression::DataBlock; use common_expression::TableDataType; @@ -79,6 +80,7 @@ impl AsyncSystemTable for StagesTable { let mut comment: Vec> = Vec::with_capacity(stages.len()); let mut number_of_files: Vec> = Vec::with_capacity(stages.len()); let mut creator: Vec>> = Vec::with_capacity(stages.len()); + let mut created_on = Vec::with_capacity(stages.len()); for stage in stages.into_iter() { name.push(stage.stage_name.clone().into_bytes()); stage_type.push(stage.stage_type.clone().to_string().into_bytes()); @@ -95,6 +97,7 @@ impl AsyncSystemTable for StagesTable { } }; creator.push(stage.creator.map(|c| c.to_string().into_bytes().to_vec())); + created_on.push(stage.created_on.timestamp_micros()); comment.push(stage.comment.clone().into_bytes()); } @@ -106,6 +109,7 @@ impl AsyncSystemTable for StagesTable { StringType::from_data(file_format_options), UInt64Type::from_opt_data(number_of_files), StringType::from_opt_data(creator), + TimestampType::from_data(created_on), StringType::from_data(comment), ])) } @@ -128,6 +132,7 @@ impl StagesTable { "creator", TableDataType::Nullable(Box::new(TableDataType::String)), ), + TableField::new("created_on", TableDataType::Timestamp), TableField::new("comment", TableDataType::String), ]); let table_info = TableInfo { diff --git a/tests/sqllogictests/suites/base/05_ddl/05_0016_ddl_stage.test b/tests/sqllogictests/suites/base/05_ddl/05_0016_ddl_stage.test index 43648d7abf55..fb92ed4d588a 100644 --- a/tests/sqllogictests/suites/base/05_ddl/05_0016_ddl_stage.test +++ b/tests/sqllogictests/suites/base/05_ddl/05_0016_ddl_stage.test @@ -19,17 +19,11 @@ CREATE STAGE test_stage_internal file_format=(type=csv compression=AUTO record_d statement ok LIST @test_stage_internal -onlyif mysql -query TTTTTITT +statement ok desc stage test_stage_internal ----- -test_stage_internal Internal StageParams { storage: Fs(StorageFsConfig { root: "_data" }) } CopyOptions { on_error: AbortNum(1), size_limit: 0, max_files: 0, split_size: 0, purge: false, single: false, max_file_size: 0, disable_variant_check: false, return_failed_only: false } Csv(CsvFileFormatParams { compression: Auto, headers: 0, field_delimiter: ",", record_delimiter: "\n", null_display: "\\N", nan_display: "NaN", escape: "\\", quote: "\"", error_on_column_count_mismatch: true }) 0 'root'@'%' (empty) -query TTTTT +statement ok SHOW STAGES ----- -test_stage External NULL 'root'@'%' (empty) -test_stage_internal Internal 0 'root'@'%' (empty) statement ok DROP STAGE test_stage diff --git a/tests/sqllogictests/suites/base/06_show/06_0011_show_stages.test b/tests/sqllogictests/suites/base/06_show/06_0011_show_stages.test index 93cb5c0a6912..5dbbdac94a09 100644 --- a/tests/sqllogictests/suites/base/06_show/06_0011_show_stages.test +++ b/tests/sqllogictests/suites/base/06_show/06_0011_show_stages.test @@ -4,16 +4,11 @@ DROP STAGE if exists test_stage statement ok CREATE STAGE test_stage -query TTITT +statement ok SHOW STAGES ----- -test_stage Internal 0 'root'@'%' (empty) - -query TTTTTITT +statement ok DESC STAGE test_stage ----- -test_stage Internal StageParams { storage: Fs(StorageFsConfig { root: "_data" }) } CopyOptions { on_error: AbortNum(1), size_limit: 0, max_files: 0, split_size: 0, purge: false, single: false, max_file_size: 0, disable_variant_check: false, return_failed_only: false } Parquet(ParquetFileFormatParams) 0 'root'@'%' (empty) statement ok DROP STAGE test_stage