feat(planner): implement explain query in verbose mode (#14801)

implement explain verbose

src/query/ast/src/ast/format/ast_format.rs

@@ -18,6 +18,7 @@ use databend_common_exception::Result;
 use databend_common_exception::Span;
 use databend_common_meta_app::principal::PrincipalIdentity;
 use databend_common_meta_app::principal::UserIdentity;
+use itertools::Itertools;
 
 use crate::ast::*;
 
@@ -699,23 +700,49 @@ impl<'ast> Visitor<'ast> for AstFormatVisitor {
         self.children.push(node);
     }
 
-    fn visit_explain(&mut self, kind: &'ast ExplainKind, query: &'ast Statement) {
+    fn visit_explain(
+        &mut self,
+        kind: &'ast ExplainKind,
+        options: &'ast [ExplainOption],
+        query: &'ast Statement,
+    ) {
         self.visit_statement(query);
         let child = self.children.pop().unwrap();
 
-        let name = format!("Explain{}", match kind {
-            ExplainKind::Ast(_) => "Ast",
-            ExplainKind::Syntax(_) => "Syntax",
-            ExplainKind::Graph => "Graph",
-            ExplainKind::Pipeline => "Pipeline",
-            ExplainKind::Fragments => "Fragments",
-            ExplainKind::Raw => "Raw",
-            ExplainKind::Optimized => "Optimized",
-            ExplainKind::Plan => "Plan",
-            ExplainKind::Memo(_) => "Memo",
-            ExplainKind::JOIN => "JOIN",
-            ExplainKind::AnalyzePlan => "Analyze",
-        });
+        let name = format!(
+            "Explain{}{}",
+            match kind {
+                ExplainKind::Ast(_) => "Ast",
+                ExplainKind::Syntax(_) => "Syntax",
+                ExplainKind::Graph => "Graph",
+                ExplainKind::Pipeline => "Pipeline",
+                ExplainKind::Fragments => "Fragments",
+                ExplainKind::Raw => "Raw",
+                ExplainKind::Optimized => "Optimized",
+                ExplainKind::Plan => "Plan",
+                ExplainKind::Memo(_) => "Memo",
+                ExplainKind::Join => "Join",
+                ExplainKind::AnalyzePlan => "Analyze",
+            },
+            if options.is_empty() {
+                "".to_string()
+            } else {
+                format!(
+                    "({})",
+                    options
+                        .iter()
+                        .flat_map(|opt| {
+                            match opt {
+                                ExplainOption::Verbose(true) => Some("Verbose"),
+                                ExplainOption::Logical(true) => Some("Logical"),
+                                ExplainOption::Optimized(true) => Some("Optimized"),
+                                _ => None,
+                            }
+                        })
+                        .join(", ")
+                )
+            }
+        );
         let format_ctx = AstFormatContext::with_children(name, 1);
         let node = FormatTreeNode::with_children(format_ctx, vec![child]);
         self.children.push(node);

src/query/ast/src/ast/statements/explain.rs

@@ -22,12 +22,23 @@ pub enum ExplainKind {
     Graph,
     Pipeline,
     Fragments,
+
+    // `EXPLAIN RAW` and `EXPLAIN OPTIMIZED` will be deprecated in the future,
+    // use explain options instead
     Raw,
     Optimized,
+
     Plan,
 
-    JOIN,
+    Join,
 
     // Explain analyze plan
     AnalyzePlan,
 }
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum ExplainOption {
+    Verbose(bool),
+    Logical(bool),
+    Optimized(bool),
+}

src/query/ast/src/ast/statements/statement.rs

@@ -19,6 +19,7 @@ use databend_common_meta_app::principal::FileFormatOptionsAst;
 use databend_common_meta_app::principal::PrincipalIdentity;
 use databend_common_meta_app::principal::UserIdentity;
 use databend_common_meta_app::schema::CreateOption;
+use itertools::Itertools;
 
 use super::merge_into::MergeIntoStmt;
 use super::*;
@@ -36,6 +37,7 @@ pub enum Statement {
     Query(Box<Query>),
     Explain {
         kind: ExplainKind,
+        options: Vec<ExplainOption>,
         query: Box<Statement>,
     },
     ExplainAnalyze {
@@ -334,8 +336,34 @@ impl Statement {
 impl Display for Statement {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         match self {
-            Statement::Explain { kind, query } => {
+            Statement::Explain {
+                options,
+                kind,
+                query,
+            } => {
                 write!(f, "EXPLAIN")?;
+                if !options.is_empty() {
+                    write!(
+                        f,
+                        "({})",
+                        options
+                            .iter()
+                            .map(|opt| {
+                                match opt {
+                                    ExplainOption::Verbose(v) => {
+                                        format!("VERBOSE = {}", v)
+                                    }
+                                    ExplainOption::Logical(v) => {
+                                        format!("LOGICAL = {}", v)
+                                    }
+                                    ExplainOption::Optimized(v) => {
+                                        format!("OPTIMIZED = {}", v)
+                                    }
+                                }
+                            })
+                            .join(", ")
+                    )?;
+                }
                 match *kind {
                     ExplainKind::Ast(_) => write!(f, " AST")?,
                     ExplainKind::Syntax(_) => write!(f, " SYNTAX")?,
@@ -346,7 +374,7 @@ impl Display for Statement {
                     ExplainKind::Optimized => write!(f, " Optimized")?,
                     ExplainKind::Plan => (),
                     ExplainKind::AnalyzePlan => write!(f, " ANALYZE")?,
-                    ExplainKind::JOIN => write!(f, " JOIN")?,
+                    ExplainKind::Join => write!(f, " JOIN")?,
                     ExplainKind::Memo(_) => write!(f, " MEMO")?,
                 }
                 write!(f, " {query}")?;

src/query/ast/src/ast/visitors/visitor.rs

@@ -380,7 +380,13 @@ pub trait Visitor<'ast>: Sized {
         walk_query(self, query);
     }
 
-    fn visit_explain(&mut self, _kind: &'ast ExplainKind, _query: &'ast Statement) {}
+    fn visit_explain(
+        &mut self,
+        _kind: &'ast ExplainKind,
+        _options: &'ast [ExplainOption],
+        _query: &'ast Statement,
+    ) {
+    }
 
     fn visit_copy_into_table(&mut self, copy: &'ast CopyIntoTableStmt) {
         if let CopyIntoTableSource::Query(query) = &copy.src {

src/query/ast/src/ast/visitors/visitor_mut.rs

@@ -394,7 +394,12 @@ pub trait VisitorMut: Sized {
         walk_query_mut(self, query);
     }
 
-    fn visit_explain(&mut self, _kind: &mut ExplainKind, stmt: &mut Statement) {
+    fn visit_explain(
+        &mut self,
+        _kind: &mut ExplainKind,
+        _options: &mut [ExplainOption],
+        stmt: &mut Statement,
+    ) {
         walk_statement_mut(self, stmt);
     }
 

src/query/ast/src/ast/visitors/walk.rs

@@ -367,7 +367,11 @@ pub fn walk_window_definition<'a, V: Visitor<'a>>(
 
 pub fn walk_statement<'a, V: Visitor<'a>>(visitor: &mut V, statement: &'a Statement) {
     match statement {
-        Statement::Explain { kind, query } => visitor.visit_explain(kind, query),
+        Statement::Explain {
+            kind,
+            options,
+            query,
+        } => visitor.visit_explain(kind, options, query),
         Statement::ExplainAnalyze { query } => visitor.visit_statement(query),
         Statement::Query(query) => visitor.visit_query(query),
         Statement::Insert(insert) => visitor.visit_insert(insert),

src/query/ast/src/ast/visitors/walk_mut.rs

@@ -362,7 +362,11 @@ pub fn walk_cte_mut<V: VisitorMut>(visitor: &mut V, cte: &mut CTE) {
 
 pub fn walk_statement_mut<V: VisitorMut>(visitor: &mut V, statement: &mut Statement) {
     match statement {
-        Statement::Explain { kind, query } => visitor.visit_explain(kind, &mut *query),
+        Statement::Explain {
+            kind,
+            options,
+            query,
+        } => visitor.visit_explain(kind, options, &mut *query),
         Statement::ExplainAnalyze { query } => visitor.visit_statement(&mut *query),
         Statement::Query(query) => visitor.visit_query(&mut *query),
         Statement::Insert(insert) => visitor.visit_insert(insert),

src/query/ast/src/parser/statement.rs

@@ -63,9 +63,9 @@ pub enum CreateDatabaseOption {
 pub fn statement(i: Input) -> IResult<StatementWithFormat> {
     let explain = map_res(
         rule! {
-            EXPLAIN ~ ( AST | SYNTAX | PIPELINE | JOIN | GRAPH | FRAGMENTS | RAW | OPTIMIZED | MEMO )? ~ #statement
+            EXPLAIN ~ ( "(" ~ #comma_separated_list1(explain_option) ~ ")" )? ~ ( AST | SYNTAX | PIPELINE | JOIN | GRAPH | FRAGMENTS | RAW | OPTIMIZED | MEMO )? ~ #statement
         },
-        |(_, opt_kind, statement)| {
+        |(_, options, opt_kind, statement)| {
             Ok(Statement::Explain {
                 kind: match opt_kind.map(|token| token.kind) {
                     Some(TokenKind::AST) => {
@@ -83,7 +83,7 @@ pub fn statement(i: Input) -> IResult<StatementWithFormat> {
                         ExplainKind::Syntax(pretty_stmt)
                     }
                     Some(TokenKind::PIPELINE) => ExplainKind::Pipeline,
-                    Some(TokenKind::JOIN) => ExplainKind::JOIN,
+                    Some(TokenKind::JOIN) => ExplainKind::Join,
                     Some(TokenKind::GRAPH) => ExplainKind::Graph,
                     Some(TokenKind::FRAGMENTS) => ExplainKind::Fragments,
                     Some(TokenKind::RAW) => ExplainKind::Raw,
@@ -92,6 +92,7 @@ pub fn statement(i: Input) -> IResult<StatementWithFormat> {
                     None => ExplainKind::Plan,
                     _ => unreachable!(),
                 },
+                options: options.as_ref().map_or(vec![], |(_, opts, _)| opts.clone()),
                 query: Box::new(statement.stmt),
             })
         },
@@ -3666,3 +3667,17 @@ pub fn alter_password_action(i: Input) -> IResult<AlterPasswordAction> {
         | #unset_options
     )(i)
 }
+
+pub fn explain_option(i: Input) -> IResult<ExplainOption> {
+    map(
+        rule! {
+            VERBOSE | LOGICAL | OPTIMIZED
+        },
+        |opt| match &opt.kind {
+            VERBOSE => ExplainOption::Verbose(true),
+            LOGICAL => ExplainOption::Logical(true),
+            OPTIMIZED => ExplainOption::Optimized(true),
+            _ => unreachable!(),
+        },
+    )(i)
+}

src/query/ast/src/parser/token.rs

@@ -673,6 +673,8 @@ pub enum TokenKind {
     LOCATION_PREFIX,
     #[token("LOCKS", ignore(ascii_case))]
     LOCKS,
+    #[token("LOGICAL", ignore(ascii_case))]
+    LOGICAL,
     #[token("ACCOUNT", ignore(ascii_case))]
     ACCOUNT,
     #[token("SECONDARY", ignore(ascii_case))]
@@ -1070,6 +1072,8 @@ pub enum TokenKind {
     VARCHAR,
     #[token("VARIANT", ignore(ascii_case))]
     VARIANT,
+    #[token("VERBOSE", ignore(ascii_case))]
+    VERBOSE,
     #[token("VIEW", ignore(ascii_case))]
     VIEW,
     #[token("VIRTUAL", ignore(ascii_case))]

src/query/ast/tests/it/parser.rs

@@ -97,6 +97,7 @@ fn test_statement() {
         r#"show create table a.b format TabSeparatedWithNamesAndTypes;"#,
         r#"explain pipeline select a from b;"#,
         r#"explain pipeline select a from t1 ignore_result;"#,
+        r#"explain(verbose, logical, optimized) select * from t where a = 1"#,
         r#"describe a;"#,
         r#"describe a format TabSeparatedWithNamesAndTypes;"#,
         r#"CREATE AGGREGATING INDEX idx1 AS SELECT SUM(a), b FROM t1 WHERE b > 3 GROUP BY b;"#,