diff --git a/Cargo.lock b/Cargo.lock index 6f2be9de2717..1f21ea115fa3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3183,6 +3183,7 @@ dependencies = [ "common-functions", "common-meta-api", "common-meta-app", + "common-meta-types", "common-metrics", "common-pipeline-core", "common-pipeline-sources", diff --git a/src/meta/api/src/schema_api_impl.rs b/src/meta/api/src/schema_api_impl.rs index 893d63620555..20e55232f38d 100644 --- a/src/meta/api/src/schema_api_impl.rs +++ b/src/meta/api/src/schema_api_impl.rs @@ -1308,15 +1308,19 @@ impl + ?Sized> SchemaApi for KV { get_pb_value(self, &req.name_ident).await?; if old_virtual_column_opt.is_some() { - return Err(KVAppError::AppError(AppError::VirtualColumnAlreadyExists( - VirtualColumnAlreadyExists::new( - req.name_ident.table_id, - format!( - "create virtual column with tenant: {} table_id: {}", - req.name_ident.tenant, req.name_ident.table_id + if req.if_not_exists { + return Ok(CreateVirtualColumnReply {}); + } else { + return Err(KVAppError::AppError(AppError::VirtualColumnAlreadyExists( + VirtualColumnAlreadyExists::new( + req.name_ident.table_id, + format!( + "create virtual column with tenant: {} table_id: {}", + req.name_ident.tenant, req.name_ident.table_id + ), ), - ), - ))); + ))); + } } let virtual_column_meta = VirtualColumnMeta { table_id: req.name_ident.table_id, @@ -1370,7 +1374,16 @@ impl + ?Sized> SchemaApi for KV { trials.next().unwrap()?; let (seq, old_virtual_column_meta) = - get_virtual_column_by_id_or_err(self, &req.name_ident, ctx).await?; + match get_virtual_column_by_id_or_err(self, &req.name_ident, ctx).await { + Ok((seq, old_virtual_column_meta)) => (seq, old_virtual_column_meta), + Err(err) => { + if req.if_exists { + return Ok(UpdateVirtualColumnReply {}); + } else { + return Err(err); + } + } + }; let virtual_column_meta = VirtualColumnMeta { table_id: req.name_ident.table_id, @@ -1423,7 +1436,13 @@ impl + ?Sized> SchemaApi for KV { loop { trials.next().unwrap()?; - let (_, _) = get_virtual_column_by_id_or_err(self, &req.name_ident, ctx).await?; + if let Err(err) = get_virtual_column_by_id_or_err(self, &req.name_ident, ctx).await { + if req.if_exists { + return Ok(DropVirtualColumnReply {}); + } else { + return Err(err); + } + } // Drop virtual column by deleting this record: // (tenant, table_id) -> virtual_column_meta diff --git a/src/meta/api/src/schema_api_test_suite.rs b/src/meta/api/src/schema_api_test_suite.rs index e60cda4a5707..e96036b869e8 100644 --- a/src/meta/api/src/schema_api_test_suite.rs +++ b/src/meta/api/src/schema_api_test_suite.rs @@ -5651,6 +5651,7 @@ impl SchemaApiTestSuite { { info!("--- create virtual column"); let req = CreateVirtualColumnReq { + if_not_exists: false, name_ident: name_ident.clone(), virtual_columns: vec!["variant:k1".to_string(), "variant[1]".to_string()], }; @@ -5659,6 +5660,7 @@ impl SchemaApiTestSuite { info!("--- create virtual column again"); let req = CreateVirtualColumnReq { + if_not_exists: false, name_ident: name_ident.clone(), virtual_columns: vec!["variant:k1".to_string(), "variant[1]".to_string()], }; @@ -5693,6 +5695,7 @@ impl SchemaApiTestSuite { { info!("--- update virtual column"); let req = UpdateVirtualColumnReq { + if_exists: false, name_ident: name_ident.clone(), virtual_columns: vec!["variant:k2".to_string(), "variant[2]".to_string()], }; @@ -5718,6 +5721,7 @@ impl SchemaApiTestSuite { { info!("--- drop virtual column"); let req = DropVirtualColumnReq { + if_exists: false, name_ident: name_ident.clone(), }; @@ -5738,6 +5742,7 @@ impl SchemaApiTestSuite { { info!("--- update virtual column after drop"); let req = UpdateVirtualColumnReq { + if_exists: false, name_ident: name_ident.clone(), virtual_columns: vec!["variant:k3".to_string(), "variant[3]".to_string()], }; diff --git a/src/meta/app/src/schema/virtual_column.rs b/src/meta/app/src/schema/virtual_column.rs index 24ef022c44ea..d1ea977ac875 100644 --- a/src/meta/app/src/schema/virtual_column.rs +++ b/src/meta/app/src/schema/virtual_column.rs @@ -56,6 +56,7 @@ pub struct VirtualColumnMeta { #[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq)] pub struct CreateVirtualColumnReq { + pub if_not_exists: bool, pub name_ident: VirtualColumnNameIdent, pub virtual_columns: Vec, } @@ -75,6 +76,7 @@ pub struct CreateVirtualColumnReply {} #[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq)] pub struct UpdateVirtualColumnReq { + pub if_exists: bool, pub name_ident: VirtualColumnNameIdent, pub virtual_columns: Vec, } @@ -94,6 +96,7 @@ pub struct UpdateVirtualColumnReply {} #[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq)] pub struct DropVirtualColumnReq { + pub if_exists: bool, pub name_ident: VirtualColumnNameIdent, } diff --git a/src/query/ast/src/ast/format/ast_format.rs b/src/query/ast/src/ast/format/ast_format.rs index c112d51155dd..81e3b14158e8 100644 --- a/src/query/ast/src/ast/format/ast_format.rs +++ b/src/query/ast/src/ast/format/ast_format.rs @@ -1839,6 +1839,32 @@ impl<'ast> Visitor<'ast> for AstFormatVisitor { self.children.push(node); } + fn visit_show_virtual_columns(&mut self, stmt: &'ast ShowVirtualColumnsStmt) { + let mut children = Vec::new(); + if let Some(database) = &stmt.database { + let database_name = format!("Database {}", database); + let database_format_ctx = AstFormatContext::new(database_name); + let database_node = FormatTreeNode::new(database_format_ctx); + children.push(database_node); + } + + if let Some(table) = &stmt.database { + let table_name = format!("Table {}", table); + let table_format_ctx = AstFormatContext::new(table_name); + let table_node = FormatTreeNode::new(table_format_ctx); + children.push(table_node); + } + + if let Some(limit) = &stmt.limit { + self.visit_show_limit(limit); + children.push(self.children.pop().unwrap()); + } + let name = "ShowVirtualColumns".to_string(); + let format_ctx = AstFormatContext::with_children(name, children.len()); + let node = FormatTreeNode::with_children(format_ctx, children); + self.children.push(node); + } + fn visit_show_users(&mut self) { let name = "ShowUsers".to_string(); let format_ctx = AstFormatContext::new(name); diff --git a/src/query/ast/src/ast/statements/statement.rs b/src/query/ast/src/ast/statements/statement.rs index 6455d38cd104..027b567e540c 100644 --- a/src/query/ast/src/ast/statements/statement.rs +++ b/src/query/ast/src/ast/statements/statement.rs @@ -157,6 +157,7 @@ pub enum Statement { AlterVirtualColumn(AlterVirtualColumnStmt), DropVirtualColumn(DropVirtualColumnStmt), RefreshVirtualColumn(RefreshVirtualColumnStmt), + ShowVirtualColumns(ShowVirtualColumnsStmt), // User ShowUsers, @@ -469,6 +470,7 @@ impl Display for Statement { Statement::AlterVirtualColumn(stmt) => write!(f, "{stmt}")?, Statement::DropVirtualColumn(stmt) => write!(f, "{stmt}")?, Statement::RefreshVirtualColumn(stmt) => write!(f, "{stmt}")?, + Statement::ShowVirtualColumns(stmt) => write!(f, "{stmt}")?, Statement::ShowUsers => write!(f, "SHOW USERS")?, Statement::ShowRoles => write!(f, "SHOW ROLES")?, Statement::CreateUser(stmt) => write!(f, "{stmt}")?, diff --git a/src/query/ast/src/ast/statements/virtual_column.rs b/src/query/ast/src/ast/statements/virtual_column.rs index 239bb2a9c78a..65f22e969672 100644 --- a/src/query/ast/src/ast/statements/virtual_column.rs +++ b/src/query/ast/src/ast/statements/virtual_column.rs @@ -19,9 +19,11 @@ use crate::ast::write_comma_separated_list; use crate::ast::write_dot_separated_list; use crate::ast::Expr; use crate::ast::Identifier; +use crate::ast::ShowLimit; #[derive(Debug, Clone, PartialEq)] pub struct CreateVirtualColumnStmt { + pub if_not_exists: bool, pub catalog: Option, pub database: Option, pub table: Identifier, @@ -31,7 +33,11 @@ pub struct CreateVirtualColumnStmt { impl Display for CreateVirtualColumnStmt { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "CREATE VIRTUAL COLUMN (")?; + write!(f, "CREATE VIRTUAL COLUMN ")?; + if self.if_not_exists { + write!(f, "IF NOT EXISTS ")?; + } + write!(f, "(")?; write_comma_separated_list(f, &self.virtual_columns)?; write!(f, ") FOR ")?; write_dot_separated_list( @@ -47,6 +53,7 @@ impl Display for CreateVirtualColumnStmt { #[derive(Debug, Clone, PartialEq)] pub struct AlterVirtualColumnStmt { + pub if_exists: bool, pub catalog: Option, pub database: Option, pub table: Identifier, @@ -56,7 +63,11 @@ pub struct AlterVirtualColumnStmt { impl Display for AlterVirtualColumnStmt { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "ALTER VIRTUAL COLUMN (")?; + write!(f, "ALTER VIRTUAL COLUMN ")?; + if self.if_exists { + write!(f, "IF EXISTS ")?; + } + write!(f, "(")?; write_comma_separated_list(f, &self.virtual_columns)?; write!(f, ") FOR ")?; write_dot_separated_list( @@ -72,6 +83,7 @@ impl Display for AlterVirtualColumnStmt { #[derive(Debug, Clone, PartialEq)] pub struct DropVirtualColumnStmt { + pub if_exists: bool, pub catalog: Option, pub database: Option, pub table: Identifier, @@ -79,7 +91,11 @@ pub struct DropVirtualColumnStmt { impl Display for DropVirtualColumnStmt { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "DROP VIRTUAL COLUMN FOR ")?; + write!(f, "DROP VIRTUAL COLUMN ")?; + if self.if_exists { + write!(f, "IF EXISTS ")?; + } + write!(f, "FOR ")?; write_dot_separated_list( f, self.catalog @@ -112,3 +128,33 @@ impl Display for RefreshVirtualColumnStmt { Ok(()) } } + +#[derive(Debug, Clone, PartialEq)] +pub struct ShowVirtualColumnsStmt { + pub catalog: Option, + pub database: Option, + pub table: Option, + pub limit: Option, +} + +impl Display for ShowVirtualColumnsStmt { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + write!(f, "SHOW VIRTUAL COLUMNS")?; + if let Some(table) = &self.table { + write!(f, "FROM {}", table)?; + } + if let Some(database) = &self.database { + write!(f, " FROM ")?; + if let Some(catalog) = &self.catalog { + write!(f, "{catalog}.",)?; + } + write!(f, "{database}")?; + } + + if let Some(limit) = &self.limit { + write!(f, " {limit}")?; + } + + Ok(()) + } +} diff --git a/src/query/ast/src/parser/statement.rs b/src/query/ast/src/parser/statement.rs index 33453f6f9082..53454501a9a6 100644 --- a/src/query/ast/src/parser/statement.rs +++ b/src/query/ast/src/parser/statement.rs @@ -958,10 +958,11 @@ pub fn statement(i: Input) -> IResult { let create_virtual_column = map( rule! { - CREATE ~ VIRTUAL ~ COLUMN ~ ^"(" ~ ^#comma_separated_list1(expr) ~ ^")" ~ FOR ~ #dot_separated_idents_1_to_3 + CREATE ~ VIRTUAL ~ COLUMN ~ ( IF ~ ^NOT ~ ^EXISTS )? ~ ^"(" ~ ^#comma_separated_list1(expr) ~ ^")" ~ FOR ~ #dot_separated_idents_1_to_3 }, - |(_, _, _, _, virtual_columns, _, _, (catalog, database, table))| { + |(_, _, _, opt_if_not_exists, _, virtual_columns, _, _, (catalog, database, table))| { Statement::CreateVirtualColumn(CreateVirtualColumnStmt { + if_not_exists: opt_if_not_exists.is_some(), catalog, database, table, @@ -972,10 +973,11 @@ pub fn statement(i: Input) -> IResult { let alter_virtual_column = map( rule! { - ALTER ~ VIRTUAL ~ COLUMN ~ ^"(" ~ ^#comma_separated_list1(expr) ~ ^")" ~ FOR ~ #dot_separated_idents_1_to_3 + ALTER ~ VIRTUAL ~ COLUMN ~ ( IF ~ ^EXISTS )? ~ ^"(" ~ ^#comma_separated_list1(expr) ~ ^")" ~ FOR ~ #dot_separated_idents_1_to_3 }, - |(_, _, _, _, virtual_columns, _, _, (catalog, database, table))| { + |(_, _, _, opt_if_exists, _, virtual_columns, _, _, (catalog, database, table))| { Statement::AlterVirtualColumn(AlterVirtualColumnStmt { + if_exists: opt_if_exists.is_some(), catalog, database, table, @@ -986,10 +988,11 @@ pub fn statement(i: Input) -> IResult { let drop_virtual_column = map( rule! { - DROP ~ VIRTUAL ~ COLUMN ~ FOR ~ #dot_separated_idents_1_to_3 + DROP ~ VIRTUAL ~ COLUMN ~ ( IF ~ ^EXISTS )? ~ FOR ~ #dot_separated_idents_1_to_3 }, - |(_, _, _, _, (catalog, database, table))| { + |(_, _, _, opt_if_exists, _, (catalog, database, table))| { Statement::DropVirtualColumn(DropVirtualColumnStmt { + if_exists: opt_if_exists.is_some(), catalog, database, table, @@ -1010,6 +1013,26 @@ pub fn statement(i: Input) -> IResult { }, ); + let show_virtual_columns = map( + rule! { + SHOW ~ VIRTUAL ~ COLUMNS ~ (( FROM | IN ) ~ #ident)? ~ (( FROM | IN ) ~ ^#dot_separated_idents_1_to_2)? ~ #show_limit? + }, + |(_, _, _, opt_table, opt_db, limit)| { + let table = opt_table.map(|(_, table)| table); + let (catalog, database) = match opt_db { + Some((_, (Some(c), d))) => (Some(c), Some(d)), + Some((_, (None, d))) => (None, Some(d)), + _ => (None, None), + }; + Statement::ShowVirtualColumns(ShowVirtualColumnsStmt { + catalog, + database, + table, + limit, + }) + }, + ); + let show_users = value(Statement::ShowUsers, rule! { SHOW ~ USERS }); let create_user = map( rule! { @@ -1739,6 +1762,7 @@ pub fn statement(i: Input) -> IResult { | #alter_virtual_column: "`ALTER VIRTUAL COLUMN (expr, ...) FOR [.]`" | #drop_virtual_column: "`DROP VIRTUAL COLUMN FOR [.]
`" | #refresh_virtual_column: "`REFRESH VIRTUAL COLUMN FOR [.]
`" + | #show_virtual_columns : "`SHOW VIRTUAL COLUMNS FROM
[FROM|IN .] []`" ), rule!( #show_users : "`SHOW USERS`" diff --git a/src/query/ast/src/visitors/visitor.rs b/src/query/ast/src/visitors/visitor.rs index 1571f5107d18..9804ec69f337 100644 --- a/src/query/ast/src/visitors/visitor.rs +++ b/src/query/ast/src/visitors/visitor.rs @@ -578,6 +578,8 @@ pub trait Visitor<'ast>: Sized { fn visit_refresh_virtual_column(&mut self, _stmt: &'ast RefreshVirtualColumnStmt) {} + fn visit_show_virtual_columns(&mut self, _stmt: &'ast ShowVirtualColumnsStmt) {} + fn visit_show_users(&mut self) {} fn visit_create_user(&mut self, _stmt: &'ast CreateUserStmt) {} diff --git a/src/query/ast/src/visitors/visitor_mut.rs b/src/query/ast/src/visitors/visitor_mut.rs index 9e1a9ccf1219..4ce7566bb3e9 100644 --- a/src/query/ast/src/visitors/visitor_mut.rs +++ b/src/query/ast/src/visitors/visitor_mut.rs @@ -592,6 +592,8 @@ pub trait VisitorMut: Sized { fn visit_refresh_virtual_column(&mut self, _stmt: &mut RefreshVirtualColumnStmt) {} + fn visit_show_virtual_columns(&mut self, _stmt: &mut ShowVirtualColumnsStmt) {} + fn visit_show_users(&mut self) {} fn visit_create_user(&mut self, _stmt: &mut CreateUserStmt) {} diff --git a/src/query/ast/src/visitors/walk.rs b/src/query/ast/src/visitors/walk.rs index 319f5e35dff0..13813a2d3ea7 100644 --- a/src/query/ast/src/visitors/walk.rs +++ b/src/query/ast/src/visitors/walk.rs @@ -436,6 +436,7 @@ pub fn walk_statement<'a, V: Visitor<'a>>(visitor: &mut V, statement: &'a Statem Statement::AlterVirtualColumn(stmt) => visitor.visit_alter_virtual_column(stmt), Statement::DropVirtualColumn(stmt) => visitor.visit_drop_virtual_column(stmt), Statement::RefreshVirtualColumn(stmt) => visitor.visit_refresh_virtual_column(stmt), + Statement::ShowVirtualColumns(stmt) => visitor.visit_show_virtual_columns(stmt), Statement::ShowUsers => visitor.visit_show_users(), Statement::ShowRoles => visitor.visit_show_roles(), Statement::CreateUser(stmt) => visitor.visit_create_user(stmt), diff --git a/src/query/ast/src/visitors/walk_mut.rs b/src/query/ast/src/visitors/walk_mut.rs index 97928f849ade..990da8deebb5 100644 --- a/src/query/ast/src/visitors/walk_mut.rs +++ b/src/query/ast/src/visitors/walk_mut.rs @@ -441,6 +441,7 @@ pub fn walk_statement_mut(visitor: &mut V, statement: &mut Statem Statement::AlterVirtualColumn(stmt) => visitor.visit_alter_virtual_column(stmt), Statement::DropVirtualColumn(stmt) => visitor.visit_drop_virtual_column(stmt), Statement::RefreshVirtualColumn(stmt) => visitor.visit_refresh_virtual_column(stmt), + Statement::ShowVirtualColumns(stmt) => visitor.visit_show_virtual_columns(stmt), Statement::ShowUsers => visitor.visit_show_users(), Statement::ShowRoles => visitor.visit_show_roles(), Statement::CreateUser(stmt) => visitor.visit_create_user(stmt), diff --git a/src/query/service/src/databases/system/system_database.rs b/src/query/service/src/databases/system/system_database.rs index e127d38089aa..1c857449e11f 100644 --- a/src/query/service/src/databases/system/system_database.rs +++ b/src/query/service/src/databases/system/system_database.rs @@ -59,6 +59,7 @@ use common_storages_system::TasksTable; use common_storages_system::TempFilesTable; use common_storages_system::TracingTable; use common_storages_system::UsersTable; +use common_storages_system::VirtualColumnsTable; use crate::catalogs::InMemoryMetas; use crate::databases::Database; @@ -125,6 +126,7 @@ impl SystemDatabase { TaskHistoryTable::create(sys_db_meta.next_table_id()), ProcessorProfileTable::create(sys_db_meta.next_table_id()), LocksTable::create(sys_db_meta.next_table_id()), + VirtualColumnsTable::create(sys_db_meta.next_table_id()), ]; let disable_tables = Self::disable_system_tables(); diff --git a/src/query/service/src/interpreters/interpreter_virtual_column_alter.rs b/src/query/service/src/interpreters/interpreter_virtual_column_alter.rs index aba09dd81dd8..39ffe4a31022 100644 --- a/src/query/service/src/interpreters/interpreter_virtual_column_alter.rs +++ b/src/query/service/src/interpreters/interpreter_virtual_column_alter.rs @@ -68,6 +68,7 @@ impl Interpreter for AlterVirtualColumnInterpreter { let catalog = self.ctx.get_catalog(&catalog_name).await?; let update_virtual_column_req = UpdateVirtualColumnReq { + if_exists: self.plan.if_exists, name_ident: VirtualColumnNameIdent { tenant, table_id }, virtual_columns: self.plan.virtual_columns.clone(), }; diff --git a/src/query/service/src/interpreters/interpreter_virtual_column_create.rs b/src/query/service/src/interpreters/interpreter_virtual_column_create.rs index 94ce0e99bc49..68ddc4e53c3c 100644 --- a/src/query/service/src/interpreters/interpreter_virtual_column_create.rs +++ b/src/query/service/src/interpreters/interpreter_virtual_column_create.rs @@ -68,6 +68,7 @@ impl Interpreter for CreateVirtualColumnInterpreter { let catalog = self.ctx.get_catalog(&catalog_name).await?; let create_virtual_column_req = CreateVirtualColumnReq { + if_not_exists: self.plan.if_not_exists, name_ident: VirtualColumnNameIdent { tenant, table_id }, virtual_columns: self.plan.virtual_columns.clone(), }; diff --git a/src/query/service/src/interpreters/interpreter_virtual_column_drop.rs b/src/query/service/src/interpreters/interpreter_virtual_column_drop.rs index a2d10a87f050..95f15838e55c 100644 --- a/src/query/service/src/interpreters/interpreter_virtual_column_drop.rs +++ b/src/query/service/src/interpreters/interpreter_virtual_column_drop.rs @@ -68,6 +68,7 @@ impl Interpreter for DropVirtualColumnInterpreter { let catalog = self.ctx.get_catalog(&catalog_name).await?; let drop_virtual_column_req = DropVirtualColumnReq { + if_exists: self.plan.if_exists, name_ident: VirtualColumnNameIdent { tenant, table_id }, }; diff --git a/src/query/sql/src/planner/binder/binder.rs b/src/query/sql/src/planner/binder/binder.rs index 6442e6feb970..3b96059c4990 100644 --- a/src/query/sql/src/planner/binder/binder.rs +++ b/src/query/sql/src/planner/binder/binder.rs @@ -321,6 +321,7 @@ impl<'a> Binder { Statement::AlterVirtualColumn(stmt) => self.bind_alter_virtual_column(stmt).await?, Statement::DropVirtualColumn(stmt) => self.bind_drop_virtual_column(stmt).await?, Statement::RefreshVirtualColumn(stmt) => self.bind_refresh_virtual_column(stmt).await?, + Statement::ShowVirtualColumns(stmt) => self.bind_show_virtual_columns(bind_context, stmt).await?, // Users Statement::CreateUser(stmt) => self.bind_create_user(stmt).await?, diff --git a/src/query/sql/src/planner/binder/ddl/virtual_column.rs b/src/query/sql/src/planner/binder/ddl/virtual_column.rs index b30807d0b35b..925c56b0fa94 100644 --- a/src/query/sql/src/planner/binder/ddl/virtual_column.rs +++ b/src/query/sql/src/planner/binder/ddl/virtual_column.rs @@ -22,17 +22,24 @@ use common_ast::ast::Expr; use common_ast::ast::Literal; use common_ast::ast::MapAccessor; use common_ast::ast::RefreshVirtualColumnStmt; +use common_ast::ast::ShowLimit; +use common_ast::ast::ShowVirtualColumnsStmt; use common_exception::ErrorCode; use common_exception::Result; use common_expression::TableDataType; use common_expression::TableSchemaRef; +use log::debug; use crate::binder::Binder; +use crate::normalize_identifier; use crate::plans::AlterVirtualColumnPlan; use crate::plans::CreateVirtualColumnPlan; use crate::plans::DropVirtualColumnPlan; use crate::plans::Plan; use crate::plans::RefreshVirtualColumnPlan; +use crate::plans::RewriteKind; +use crate::BindContext; +use crate::SelectBuilder; impl Binder { #[async_backtrace::framed] @@ -41,6 +48,7 @@ impl Binder { stmt: &CreateVirtualColumnStmt, ) -> Result { let CreateVirtualColumnStmt { + if_not_exists, catalog, database, table, @@ -64,6 +72,7 @@ impl Binder { Ok(Plan::CreateVirtualColumn(Box::new( CreateVirtualColumnPlan { + if_not_exists: *if_not_exists, catalog, database, table, @@ -78,6 +87,7 @@ impl Binder { stmt: &AlterVirtualColumnStmt, ) -> Result { let AlterVirtualColumnStmt { + if_exists, catalog, database, table, @@ -100,6 +110,7 @@ impl Binder { .await?; Ok(Plan::AlterVirtualColumn(Box::new(AlterVirtualColumnPlan { + if_exists: *if_exists, catalog, database, table, @@ -113,6 +124,7 @@ impl Binder { stmt: &DropVirtualColumnStmt, ) -> Result { let DropVirtualColumnStmt { + if_exists, catalog, database, table, @@ -129,6 +141,7 @@ impl Binder { } Ok(Plan::DropVirtualColumn(Box::new(DropVirtualColumnPlan { + if_exists: *if_exists, catalog, database, table, @@ -246,4 +259,66 @@ impl Binder { Ok(virtual_columns) } + + #[async_backtrace::framed] + pub(in crate::planner::binder) async fn bind_show_virtual_columns( + &mut self, + bind_context: &mut BindContext, + stmt: &ShowVirtualColumnsStmt, + ) -> Result { + let ShowVirtualColumnsStmt { + catalog, + database, + table, + limit, + } = stmt; + + let catalog_name = match catalog { + None => self.ctx.get_current_catalog(), + Some(ident) => { + let catalog = normalize_identifier(ident, &self.name_resolution_ctx).name; + self.ctx.get_catalog(&catalog).await?; + catalog + } + }; + let catalog = self.ctx.get_catalog(&catalog_name).await?; + let database = match database { + None => self.ctx.get_current_database(), + Some(ident) => { + let database = normalize_identifier(ident, &self.name_resolution_ctx).name; + catalog + .get_database(&self.ctx.get_tenant(), &database) + .await?; + database + } + }; + + let mut select_builder = SelectBuilder::from("system.virtual_columns"); + select_builder + .with_column("database") + .with_column("table") + .with_column("virtual_columns"); + + select_builder.with_filter(format!("database = '{database}'")); + if let Some(table) = table { + let table = normalize_identifier(table, &self.name_resolution_ctx).name; + select_builder.with_filter(format!("table = '{table}'")); + } + + let query = match limit { + None => select_builder.build(), + Some(ShowLimit::Like { pattern }) => { + select_builder.with_filter(format!("virtual_columns LIKE '{pattern}'")); + select_builder.build() + } + Some(ShowLimit::Where { selection }) => { + select_builder.with_filter(format!("({selection})")); + select_builder.build() + } + }; + debug!("show virtual columns rewrite to: {:?}", query); + + self.bind_rewrite_to_query(bind_context, &query, RewriteKind::ShowVirtualColumns) + .await + } } diff --git a/src/query/sql/src/planner/plans/ddl/virtual_column.rs b/src/query/sql/src/planner/plans/ddl/virtual_column.rs index eace72f356e8..4b49a9e384f5 100644 --- a/src/query/sql/src/planner/plans/ddl/virtual_column.rs +++ b/src/query/sql/src/planner/plans/ddl/virtual_column.rs @@ -20,6 +20,7 @@ use storages_common_table_meta::meta::Location; #[derive(Clone, Debug, PartialEq, Eq)] pub struct CreateVirtualColumnPlan { + pub if_not_exists: bool, pub catalog: String, pub database: String, pub table: String, @@ -34,6 +35,7 @@ impl CreateVirtualColumnPlan { #[derive(Clone, Debug, PartialEq, Eq)] pub struct AlterVirtualColumnPlan { + pub if_exists: bool, pub catalog: String, pub database: String, pub table: String, @@ -48,6 +50,7 @@ impl AlterVirtualColumnPlan { #[derive(Clone, Debug, PartialEq, Eq)] pub struct DropVirtualColumnPlan { + pub if_exists: bool, pub catalog: String, pub database: String, pub table: String, diff --git a/src/query/sql/src/planner/plans/plan.rs b/src/query/sql/src/planner/plans/plan.rs index 4dc958c43dba..c0baf978d935 100644 --- a/src/query/sql/src/planner/plans/plan.rs +++ b/src/query/sql/src/planner/plans/plan.rs @@ -325,6 +325,7 @@ pub enum RewriteKind { ShowTables(String), ShowColumns(String, String), ShowTablesStatus, + ShowVirtualColumns, ShowStreams(String), diff --git a/src/query/storages/system/Cargo.toml b/src/query/storages/system/Cargo.toml index ed812b0be770..d0e0912ed95c 100644 --- a/src/query/storages/system/Cargo.toml +++ b/src/query/storages/system/Cargo.toml @@ -22,6 +22,7 @@ common-expression = { path = "../../expression" } common-functions = { path = "../../functions" } common-meta-api = { path = "../../../meta/api" } common-meta-app = { path = "../../../meta/app" } +common-meta-types = { path = "../../../meta/types" } common-metrics = { path = "../../../common/metrics" } common-pipeline-core = { path = "../../pipeline/core" } common-pipeline-sources = { path = "../../pipeline/sources" } diff --git a/src/query/storages/system/src/columns_table.rs b/src/query/storages/system/src/columns_table.rs index ab3dd7de2e15..4baec96b43b7 100644 --- a/src/query/storages/system/src/columns_table.rs +++ b/src/query/storages/system/src/columns_table.rs @@ -143,87 +143,98 @@ impl ColumnsTable { ctx: Arc, push_downs: Option, ) -> Result> { - let tenant = ctx.get_tenant(); - let catalog = ctx.get_catalog(CATALOG_DEFAULT).await?; - - let mut tables = Vec::new(); - let mut databases = Vec::new(); - - if let Some(push_downs) = push_downs { - if let Some(filter) = push_downs.filters.as_ref().map(|f| &f.filter) { - let expr = filter.as_expr(&BUILTIN_FUNCTIONS); - find_eq_filter(&expr, &mut |col_name, scalar| { - if col_name == "database" { - if let Scalar::String(s) = scalar { - if let Ok(database) = String::from_utf8(s.clone()) { - if !databases.contains(&database) { - databases.push(database); - } + let database_and_tables = dump_tables(&ctx, push_downs).await?; + + let mut rows: Vec<(String, String, TableField)> = vec![]; + for (database, tables) in database_and_tables { + for table in tables { + let fields = generate_fields(&ctx, &table).await?; + for field in fields { + rows.push((database.clone(), table.name().into(), field.clone())) + } + } + } + + Ok(rows) + } +} + +pub(crate) async fn dump_tables( + ctx: &Arc, + push_downs: Option, +) -> Result>)>> { + let tenant = ctx.get_tenant(); + let catalog = ctx.get_catalog(CATALOG_DEFAULT).await?; + + let mut tables = Vec::new(); + let mut databases = Vec::new(); + + if let Some(push_downs) = push_downs { + if let Some(filter) = push_downs.filters.as_ref().map(|f| &f.filter) { + let expr = filter.as_expr(&BUILTIN_FUNCTIONS); + find_eq_filter(&expr, &mut |col_name, scalar| { + if col_name == "database" { + if let Scalar::String(s) = scalar { + if let Ok(database) = String::from_utf8(s.clone()) { + if !databases.contains(&database) { + databases.push(database); } } - } else if col_name == "table" { - if let Scalar::String(s) = scalar { - if let Ok(table) = String::from_utf8(s.clone()) { - if !tables.contains(&table) { - tables.push(table); - } + } + } else if col_name == "table" { + if let Scalar::String(s) = scalar { + if let Ok(table) = String::from_utf8(s.clone()) { + if !tables.contains(&table) { + tables.push(table); } } } - }); - } + } + }); } + } - if databases.is_empty() { - let all_databases = catalog.list_databases(tenant.as_str()).await?; - for db in all_databases { - databases.push(db.name().to_string()); - } + if databases.is_empty() { + let all_databases = catalog.list_databases(tenant.as_str()).await?; + for db in all_databases { + databases.push(db.name().to_string()); } + } - let tenant = ctx.get_tenant(); - let visibility_checker = ctx.get_visibility_checker().await?; + let visibility_checker = ctx.get_visibility_checker().await?; - let final_dbs: Vec = databases - .iter() - .filter(|db| visibility_checker.check_database_visibility(CATALOG_DEFAULT, db)) - .cloned() - .collect(); + let final_dbs: Vec = databases + .iter() + .filter(|db| visibility_checker.check_database_visibility(CATALOG_DEFAULT, db)) + .cloned() + .collect(); - let mut rows: Vec<(String, String, TableField)> = vec![]; - for database in final_dbs { - let tables = if tables.is_empty() { - if let Ok(table) = catalog.list_tables(tenant.as_str(), &database).await { - table - } else { - vec![] - } + let mut final_tables: Vec<(String, Vec>)> = Vec::with_capacity(final_dbs.len()); + for database in final_dbs { + let tables = if tables.is_empty() { + if let Ok(table) = catalog.list_tables(tenant.as_str(), &database).await { + table } else { - let mut res = Vec::new(); - for table in &tables { - if let Ok(table) = catalog.get_table(tenant.as_str(), &database, table).await { - res.push(table); - } - } - res - }; - - for table in tables { - if visibility_checker.check_table_visibility( - CATALOG_DEFAULT, - &database, - table.name(), - ) { - let fields = generate_fields(&ctx, &table).await?; - for field in fields { - rows.push((database.clone(), table.name().into(), field.clone())) - } + vec![] + } + } else { + let mut res = Vec::new(); + for table in &tables { + if let Ok(table) = catalog.get_table(tenant.as_str(), &database, table).await { + res.push(table); } } + res + }; + let mut filtered_tables = Vec::with_capacity(tables.len()); + for table in tables { + if visibility_checker.check_table_visibility(CATALOG_DEFAULT, &database, table.name()) { + filtered_tables.push(table); + } } - - Ok(rows) + final_tables.push((database, filtered_tables)); } + Ok(final_tables) } async fn generate_fields( diff --git a/src/query/storages/system/src/lib.rs b/src/query/storages/system/src/lib.rs index bcf3fe7e17f2..977884955a9b 100644 --- a/src/query/storages/system/src/lib.rs +++ b/src/query/storages/system/src/lib.rs @@ -59,6 +59,7 @@ mod temp_files_table; mod tracing_table; mod users_table; mod util; +mod virtual_columns_table; pub use background_jobs_table::BackgroundJobTable; pub use background_tasks_table::BackgroundTaskTable; @@ -112,3 +113,4 @@ pub use tasks_table::TasksTable; pub use temp_files_table::TempFilesTable; pub use tracing_table::TracingTable; pub use users_table::UsersTable; +pub use virtual_columns_table::VirtualColumnsTable; diff --git a/src/query/storages/system/src/virtual_columns_table.rs b/src/query/storages/system/src/virtual_columns_table.rs new file mode 100644 index 000000000000..934c64647433 --- /dev/null +++ b/src/query/storages/system/src/virtual_columns_table.rs @@ -0,0 +1,138 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; +use std::sync::Arc; + +use common_catalog::catalog::CATALOG_DEFAULT; +use common_catalog::plan::PushDownInfo; +use common_catalog::table::Table; +use common_exception::Result; +use common_expression::types::StringType; +use common_expression::types::TimestampType; +use common_expression::DataBlock; +use common_expression::FromData; +use common_expression::TableDataType; +use common_expression::TableField; +use common_expression::TableSchemaRefExt; +use common_meta_app::schema::ListVirtualColumnsReq; +use common_meta_app::schema::TableIdent; +use common_meta_app::schema::TableInfo; +use common_meta_app::schema::TableMeta; +use common_meta_app::schema::VirtualColumnMeta; +use common_meta_types::MetaId; +use common_storages_fuse::TableContext; + +use crate::columns_table::dump_tables; +use crate::table::AsyncOneBlockSystemTable; +use crate::table::AsyncSystemTable; + +pub struct VirtualColumnsTable { + table_info: TableInfo, +} + +#[async_trait::async_trait] +impl AsyncSystemTable for VirtualColumnsTable { + const NAME: &'static str = "system.virtual_columns"; + + fn get_table_info(&self) -> &TableInfo { + &self.table_info + } + + async fn get_full_data( + &self, + ctx: Arc, + push_downs: Option, + ) -> Result { + let tenant = ctx.get_tenant(); + let catalog = ctx.get_catalog(CATALOG_DEFAULT).await?; + let virtual_column_metas = catalog + .list_virtual_columns(ListVirtualColumnsReq { + tenant: tenant.clone(), + table_id: None, + }) + .await?; + + let mut database_names = Vec::with_capacity(virtual_column_metas.len()); + let mut table_names = Vec::with_capacity(virtual_column_metas.len()); + let mut virtual_columns = Vec::with_capacity(virtual_column_metas.len()); + let mut created_ons = Vec::with_capacity(virtual_column_metas.len()); + let mut updated_ons = Vec::with_capacity(virtual_column_metas.len()); + if !virtual_column_metas.is_empty() { + let mut virtual_column_meta_map: HashMap = + virtual_column_metas + .into_iter() + .map(|v| (v.table_id, v)) + .collect(); + + let database_and_tables = dump_tables(&ctx, push_downs).await?; + for (database, tables) in database_and_tables { + for table in tables { + let table_id = table.get_id(); + if let Some(virtual_column_meta) = virtual_column_meta_map.remove(&table_id) { + database_names.push(database.as_bytes().to_vec()); + table_names.push(table.name().as_bytes().to_vec()); + virtual_columns.push( + virtual_column_meta + .virtual_columns + .join(", ") + .as_bytes() + .to_vec(), + ); + created_ons.push(virtual_column_meta.created_on.timestamp_micros()); + updated_ons + .push(virtual_column_meta.updated_on.map(|u| u.timestamp_micros())); + } + } + } + } + + Ok(DataBlock::new_from_columns(vec![ + StringType::from_data(database_names), + StringType::from_data(table_names), + StringType::from_data(virtual_columns), + TimestampType::from_data(created_ons), + TimestampType::from_opt_data(updated_ons), + ])) + } +} + +impl VirtualColumnsTable { + pub fn create(table_id: u64) -> Arc { + let schema = TableSchemaRefExt::create(vec![ + TableField::new("database", TableDataType::String), + TableField::new("table", TableDataType::String), + TableField::new("virtual_columns", TableDataType::String), + TableField::new("created_on", TableDataType::Timestamp), + TableField::new( + "updated_on", + TableDataType::Nullable(Box::new(TableDataType::Timestamp)), + ), + ]); + + let table_info = TableInfo { + desc: "'system'.'virtual_columns'".to_string(), + name: "virtual_columns".to_string(), + ident: TableIdent::new(table_id, 0), + meta: TableMeta { + schema, + engine: "SystemVirtualColumns".to_string(), + ..Default::default() + }, + ..Default::default() + }; + + AsyncOneBlockSystemTable::create(Self { table_info }) + } +} diff --git a/tests/sqllogictests/suites/ee/01_ee_system/01_0002_virtual_column.test b/tests/sqllogictests/suites/ee/01_ee_system/01_0002_virtual_column.test index 80c8346371fb..cfd184064e60 100644 --- a/tests/sqllogictests/suites/ee/01_ee_system/01_0002_virtual_column.test +++ b/tests/sqllogictests/suites/ee/01_ee_system/01_0002_virtual_column.test @@ -39,6 +39,11 @@ create virtual column (val['a'], val['b']) for t1 statement ok refresh virtual column for t1 +query TTT +show virtual columns from t1 +---- +test_virtual_column t1 val['a'], val['b'] + statement ok insert into t1 values(4, '{"a":44,"b":4}'), (5, '{"a":55}'), (6, '6') @@ -138,6 +143,11 @@ create virtual column (val['a'], val['b']) for t2 statement ok refresh virtual column for t2 +query TTT +show virtual columns from t2 +---- +test_virtual_column t2 val['a'], val['b'] + statement ok insert into t2 values(4, '{"a":44,"b":4}'), (5, '{"a":55}'), (6, '6') diff --git a/tests/sqllogictests/suites/ee/05_ee_ddl/05_0002_ddl_create_drop_virtual_columns.test b/tests/sqllogictests/suites/ee/05_ee_ddl/05_0002_ddl_create_drop_virtual_columns.test index 3b23cc3b314b..77dcc2e5737c 100644 --- a/tests/sqllogictests/suites/ee/05_ee_ddl/05_0002_ddl_create_drop_virtual_columns.test +++ b/tests/sqllogictests/suites/ee/05_ee_ddl/05_0002_ddl_create_drop_virtual_columns.test @@ -30,9 +30,15 @@ CREATE TABLE t1(a int, v json) Engine = Fuse statement error 1115 ALTER VIRTUAL COLUMN (v['k1'], v:k2, v[0]) FOR t1; +statement ok +ALTER VIRTUAL COLUMN IF EXISTS (v['k1'], v:k2) FOR t1; + statement error 1115 DROP VIRTUAL COLUMN FOR t1; +statement ok +DROP VIRTUAL COLUMN IF EXISTS FOR t1; + statement error 1065 CREATE VIRTUAL COLUMN (v) FOR t1; @@ -42,6 +48,9 @@ CREATE VIRTUAL COLUMN (a['k1']) FOR t1; statement ok CREATE VIRTUAL COLUMN (v['k1'], v:k2, v[0]) FOR t1; +statement ok +CREATE VIRTUAL COLUMN IF NOT EXISTS (v['k1'], v:k2, v[0]) FOR t1; + statement error 1116 CREATE VIRTUAL COLUMN (v['k1'], v:k2, v[0]) FOR t1;