Merge branch 'main' into lock_manager

docs/doc/13-sql-reference/51-connect-parameters.md

@@ -14,7 +14,7 @@ The connection parameters are specified using a CONNECTION clause and are separa
 ```sql title='Examples:'
 -- This example illustrates a 'CREATE STAGE' command where 'CONNECTION' is followed by '=', establishing a Minio stage with specific connection parameters.
 CREATE STAGE my_minio_stage
-  URL = 's3://databend'
+  's3://databend'
   CONNECTION = (
     ENDPOINT_URL = 'http://localhost:9000',
     ACCESS_KEY_ID = 'ROOTUSER',

docs/doc/14-sql-commands/00-ddl/40-stage/01-ddl-create-stage.md

@@ -38,7 +38,7 @@ import TabItem from '@theme/TabItem';
 
 ```sql
 externalStageParams ::=
-  URL = 's3://<bucket>[<path/>]'
+  's3://<bucket>[<path/>]'
   CONNECTION = (
         <connection_parameters>
   )
@@ -55,7 +55,7 @@ To create an external stage on Amazon S3, you can also use an IAM user account,
 
 ```sql
 externalStageParams ::=
-  URL = 'azblob://<container>[<path/>]'
+  'azblob://<container>[<path/>]'
   CONNECTION = (
         <connection_parameters>
   )
@@ -68,7 +68,7 @@ For the connection parameters available for accessing Azure Blob Storage, see [C
 
 ```sql
 externalLocation ::=
-  URL = 'gcs://<bucket>[<path>]'
+  'gcs://<bucket>[<path>]'
   CONNECTION = (
         <connection_parameters>
   )
@@ -81,7 +81,7 @@ For the connection parameters available for accessing Google Cloud Storage, see
 
 ```sql
 externalLocation ::=
-  URL = 'oss://<bucket>[<path>]'
+  'oss://<bucket>[<path>]'
   CONNECTION = (
         <connection_parameters>
   )
@@ -94,7 +94,7 @@ For the connection parameters available for accessing Alibaba Cloud OSS, see [Co
 
 ```sql
 externalLocation ::=
-  URL = 'cos://<bucket>[<path>]'
+  'cos://<bucket>[<path>]'
   CONNECTION = (
         <connection_parameters>
   )
@@ -107,7 +107,7 @@ For the connection parameters available for accessing Tencent Cloud Object Stora
 
 ```sql
 externalLocation ::=
-  URL = "hdfs://<endpoint_url>[<path>]"
+  "hdfs://<endpoint_url>[<path>]"
   CONNECTION = (
         <connection_parameters>
   )
@@ -120,7 +120,7 @@ For the connection parameters available for accessing HDFS, see [Connection Para
 
 ```sql
 externalLocation ::=
-  URL = "webhdfs://<endpoint_url>[<path>]"
+  "webhdfs://<endpoint_url>[<path>]"
   CONNECTION = (
         <connection_parameters>
   )

docs/doc/90-contributing/01-rfcs/20221013-iceberg-catalog.md

@@ -181,10 +181,10 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name
     ...
 ) ENGINE = `ICEBERG`
 ENGINE_OPTIONS = (
-URL = 's3://path/to/iceberg'
-DATABASE = <iceberg_db>
-TABLE = <iceberg_tbl>
-[ SNAPSHOT = { SNAPSHOT_ID => <snapshot_id> | TIMESTAMP => <timestamp> } ]
+  URL = 's3://path/to/iceberg'
+  DATABASE = <iceberg_db>
+  TABLE = <iceberg_tbl>
+  [ SNAPSHOT = { SNAPSHOT_ID => <snapshot_id> | TIMESTAMP => <timestamp> } ]
 )
 ```
 

src/common/base/Cargo.toml

@@ -47,6 +47,7 @@ pprof = { version = "0.11.1", features = [
     "protobuf-codec",
     "protobuf",
 ] }
+regex = "1.8.1"
 semver = { workspace = true }
 serde = { workspace = true }
 state = "0.5"

src/common/base/src/base/mod.rs

@@ -42,6 +42,7 @@ pub use stoppable::Stoppable;
 pub use string::convert_byte_size;
 pub use string::convert_number_size;
 pub use string::escape_for_key;
+pub use string::mask_connection_info;
 pub use string::mask_string;
 pub use string::unescape_for_key;
 pub use string::unescape_string;

src/common/base/src/base/string.rs

@@ -16,6 +16,7 @@ use std::string::FromUtf8Error;
 
 use common_exception::ErrorCode;
 use common_exception::Result;
+use regex::Regex;
 
 /// Function that escapes special characters in a string.
 ///
@@ -162,3 +163,18 @@ pub fn convert_number_size(num: f64) -> String {
     let unit = units[exponent as usize];
     format!("{}{}{}", negative, pretty_bytes, unit)
 }
+
+/// Mask the connection info in the sql.
+pub fn mask_connection_info(sql: &str) -> String {
+    let mut masked_sql = sql.to_string();
+
+    // Regular expression to find the CONNECTION block
+    let re_connection = Regex::new(r"CONNECTION\s*=\s*\([^)]+\)").unwrap();
+
+    // Replace the entire CONNECTION block with 'CONNECTION = (***masked***)'
+    masked_sql = re_connection
+        .replace_all(&masked_sql, "CONNECTION = (***masked***)")
+        .to_string();
+
+    masked_sql
+}

src/common/base/tests/it/main.rs

@@ -22,7 +22,7 @@ mod range_merger;
 mod runtime;
 mod runtime_tracker;
 mod stoppable;
-mod string_func;
+mod string;
 mod thread_pool;
 
 // runtime tests depends on the memory stat collector.

src/common/base/tests/it/string_func.rs → src/common/base/tests/it/string.rs

@@ -75,3 +75,35 @@ fn test_unescape_string() {
         assert_eq!(unescape_string(c[0]).unwrap(), c[1]);
     }
 }
+
+#[test]
+fn test_mask_connection_info() {
+    let sql = r#"COPY INTO table1 
+        FROM 's3://xx/yy
+        CONNECTION = (
+            ACCESS_KEY_ID = 'aaa' ,
+            SECRET_ACCESS_KEY = 'sss' ,
+            REGION = 'us-east-2'
+        )
+        PATTERN = '.*[.]csv'
+            FILE_FORMAT = (
+            TYPE = CSV,
+            FIELD_DELIMITER = '\t',
+            RECORD_DELIMITER = '\n',
+            SKIP_HEADER = 1
+        );"#;
+
+    let actual = mask_connection_info(sql);
+    let expect = r#"COPY INTO table1 
+        FROM 's3://xx/yy
+        CONNECTION = (***masked***)
+        PATTERN = '.*[.]csv'
+            FILE_FORMAT = (
+            TYPE = CSV,
+            FIELD_DELIMITER = '\t',
+            RECORD_DELIMITER = '\n',
+            SKIP_HEADER = 1
+        );"#;
+
+    assert_eq!(expect, actual);
+}

src/meta/app/src/principal/user_stage.rs

@@ -49,7 +49,7 @@ use crate::storage::StorageParams;
 // WHERE
 //
 // externalStageParams (for Amazon S3) ::=
-// URL = 's3://<bucket>[/<path>/]'
+// 's3://<bucket>[/<path>/]'
 // [ { CREDENTIALS = ( {  { AWS_KEY_ID = '<string>' AWS_SECRET_KEY = '<string>' [ AWS_TOKEN = '<string>' ] } | AWS_ROLE = '<string>'  } ) ) } ]
 //
 // copyOptions ::=

src/query/ast/src/ast/format/ast_format.rs

@@ -910,48 +910,48 @@ impl<'ast> Visitor<'ast> for AstFormatVisitor {
         self.children.push(node);
     }
 
-    fn visit_show_settings(&mut self, like: &'ast Option<String>) {
-        let mut children = Vec::new();
-        if let Some(like) = like {
-            let like_name = format!("Like {}", like);
-            let like_format_ctx = AstFormatContext::new(like_name);
-            let like_node = FormatTreeNode::new(like_format_ctx);
-            children.push(like_node);
-        }
-        let name = "ShowSetting".to_string();
-        let format_ctx = AstFormatContext::with_children(name, children.len());
-        let node = FormatTreeNode::with_children(format_ctx, children);
-        self.children.push(node);
+    fn visit_show_settings(&mut self, show_options: &'ast Option<ShowOptions>) {
+        self.visit_show_options(show_options, "ShowSetting".to_string());
     }
 
-    fn visit_show_process_list(&mut self) {
-        let name = "ShowProcessList".to_string();
-        let format_ctx = AstFormatContext::new(name);
-        let node = FormatTreeNode::new(format_ctx);
-        self.children.push(node);
+    fn visit_show_process_list(&mut self, show_options: &'ast Option<ShowOptions>) {
+        self.visit_show_options(show_options, "ShowProcessList".to_string());
     }
 
-    fn visit_show_metrics(&mut self) {
-        let name = "ShowMetrics".to_string();
-        let format_ctx = AstFormatContext::new(name);
-        let node = FormatTreeNode::new(format_ctx);
-        self.children.push(node);
+    fn visit_show_metrics(&mut self, show_options: &'ast Option<ShowOptions>) {
+        self.visit_show_options(show_options, "ShowMetrics".to_string());
     }
 
-    fn visit_show_engines(&mut self) {
-        let name = "ShowEngines".to_string();
-        let format_ctx = AstFormatContext::new(name);
-        let node = FormatTreeNode::new(format_ctx);
-        self.children.push(node);
+    fn visit_show_engines(&mut self, show_options: &'ast Option<ShowOptions>) {
+        self.visit_show_options(show_options, "ShowEngines".to_string());
+    }
+
+    fn visit_show_functions(&mut self, show_options: &'ast Option<ShowOptions>) {
+        self.visit_show_options(show_options, "ShowFunctions".to_string());
+    }
+
+    fn visit_show_table_functions(&mut self, show_options: &'ast Option<ShowOptions>) {
+        self.visit_show_options(show_options, "ShowTableFunctions".to_string());
     }
 
-    fn visit_show_functions(&mut self, limit: &'ast Option<ShowLimit>) {
+    fn visit_show_indexes(&mut self, show_options: &'ast Option<ShowOptions>) {
+        self.visit_show_options(show_options, "ShowIndexes".to_string());
+    }
+
+    fn visit_show_options(&mut self, show_options: &'ast Option<ShowOptions>, name: String) {
         let mut children = Vec::new();
-        if let Some(limit) = limit {
-            self.visit_show_limit(limit);
-            children.push(self.children.pop().unwrap());
+        if let Some(show_options) = show_options {
+            if let Some(show_limit) = &show_options.show_limit {
+                self.visit_show_limit(show_limit);
+                children.push(self.children.pop().unwrap());
+            }
+            if let Some(limit) = show_options.limit {
+                let name = format!("Limit {}", limit);
+                let limit_format_ctx = AstFormatContext::new(name);
+                let node = FormatTreeNode::new(limit_format_ctx);
+                children.push(node);
+            }
         }
-        let name = "ShowFunctions".to_string();
         let format_ctx = AstFormatContext::with_children(name, children.len());
         let node = FormatTreeNode::with_children(format_ctx, children);
         self.children.push(node);

src/query/ast/src/ast/statements/show.rs

@@ -31,3 +31,21 @@ impl Display for ShowLimit {
         }
     }
 }
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct ShowOptions {
+    pub show_limit: Option<ShowLimit>,
+    pub limit: Option<u64>,
+}
+
+impl Display for ShowOptions {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        if let Some(limit_option) = self.show_limit.clone() {
+            write!(f, "{}", limit_option)?;
+        }
+        if let Some(limit) = self.limit {
+            write!(f, " LIMIT {limit}")?;
+        }
+        Ok(())
+    }
+}

src/query/ast/src/ast/statements/stage.rs

@@ -44,8 +44,7 @@ impl Display for CreateStageStmt {
         write!(f, " {}", self.stage_name)?;
 
         if let Some(ul) = &self.location {
-            write!(f, " URL = ")?;
-            write!(f, "{ul}")?;
+            write!(f, " {ul}")?;
         }
 
         if !self.file_format_options.is_empty() {