diff --git a/src/query/ast/src/ast/ast.rs b/src/query/ast/src/ast/ast.rs index cde3b5b9ff9f9..843d46c3b0ca0 100644 --- a/src/query/ast/src/ast/ast.rs +++ b/src/query/ast/src/ast/ast.rs @@ -114,7 +114,7 @@ pub(crate) fn write_comma_separated_list( } /// Write input items into `'a', 'b', 'c'` -pub(crate) fn write_quoted_comma_separated_list( +pub(crate) fn write_comma_separated_quoted_list( f: &mut Formatter<'_>, items: impl IntoIterator, ) -> std::fmt::Result { @@ -127,16 +127,30 @@ pub(crate) fn write_quoted_comma_separated_list( Ok(()) } -/// Write input map items into `field_a=x field_b=y` -pub(crate) fn write_space_separated_map( +/// Write input map items into `field_a=x, field_b=y` +pub(crate) fn write_comma_separated_map( f: &mut Formatter<'_>, items: impl IntoIterator, ) -> std::fmt::Result { for (i, (k, v)) in items.into_iter().enumerate() { if i > 0 { - write!(f, " ")?; + write!(f, ", ")?; } write!(f, "{k}='{v}'")?; } Ok(()) } + +/// Write input map items into `field_a=>x, field_b=>y` +pub(crate) fn write_comma_separated_arrow_map( + f: &mut Formatter<'_>, + items: impl IntoIterator, +) -> std::fmt::Result { + for (i, (k, v)) in items.into_iter().enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "{k}=>'{v}'")?; + } + Ok(()) +} diff --git a/src/query/ast/src/ast/statements/catalog.rs b/src/query/ast/src/ast/statements/catalog.rs index e1d99a4b7abaa..bd8d08c74c7ce 100644 --- a/src/query/ast/src/ast/statements/catalog.rs +++ b/src/query/ast/src/ast/statements/catalog.rs @@ -19,6 +19,7 @@ use std::fmt::Formatter; use common_meta_app::schema::CatalogType; use super::ShowLimit; +use crate::ast::write_comma_separated_map; use crate::ast::Identifier; #[derive(Debug, Clone, PartialEq)] @@ -64,10 +65,8 @@ impl Display for CreateCatalogStmt { } write!(f, " {}", self.catalog_name)?; write!(f, " TYPE='{}'", self.catalog_type)?; - write!(f, " CONNECTION = (")?; - for (k, v) in self.catalog_options.iter() { - write!(f, " {}='{}'", k, v)?; - } + write!(f, " CONNECTION = ( ")?; + write_comma_separated_map(f, &self.catalog_options)?; write!(f, " )") } } diff --git a/src/query/ast/src/ast/statements/copy.rs b/src/query/ast/src/ast/statements/copy.rs index afa85b33754a2..0eb5eeee7599a 100644 --- a/src/query/ast/src/ast/statements/copy.rs +++ b/src/query/ast/src/ast/statements/copy.rs @@ -23,8 +23,8 @@ use std::io::Result; use itertools::Itertools; use url::Url; -use crate::ast::write_quoted_comma_separated_list; -use crate::ast::write_space_separated_map; +use crate::ast::write_comma_separated_map; +use crate::ast::write_comma_separated_quoted_list; use crate::ast::Hint; use crate::ast::Identifier; use crate::ast::Query; @@ -88,7 +88,7 @@ impl Display for CopyStmt { if let Some(files) = &self.files { write!(f, " FILES = (")?; - write_quoted_comma_separated_list(f, files)?; + write_comma_separated_quoted_list(f, files)?; write!(f, " )")?; } @@ -98,10 +98,8 @@ impl Display for CopyStmt { if !self.file_format.is_empty() { write!(f, " FILE_FORMAT = (")?; - for (k, v) in self.file_format.iter() { - write!(f, " {} = '{}'", k, v)?; - } - write!(f, " )")?; + write_comma_separated_map(f, &self.file_format)?; + write!(f, ")")?; } if !self.validation_mode.is_empty() { @@ -256,7 +254,7 @@ impl Display for Connection { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { if !self.conns.is_empty() { write!(f, " CONNECTION = ( ")?; - write_space_separated_map(f, &self.conns)?; + write_comma_separated_map(f, &self.conns)?; write!(f, " )")?; } Ok(()) diff --git a/src/query/ast/src/ast/statements/insert.rs b/src/query/ast/src/ast/statements/insert.rs index d3ab868c6db78..78a5ff3d2ad9e 100644 --- a/src/query/ast/src/ast/statements/insert.rs +++ b/src/query/ast/src/ast/statements/insert.rs @@ -17,6 +17,7 @@ use std::fmt::Display; use std::fmt::Formatter; use crate::ast::write_comma_separated_list; +use crate::ast::write_comma_separated_map; use crate::ast::write_dot_separated_list; use crate::ast::Hint; use crate::ast::Identifier; @@ -95,9 +96,7 @@ impl Display for InsertSource { start: _, } => { write!(f, " FILE_FORMAT = (")?; - for (k, v) in settings.iter() { - write!(f, " {} = '{}'", k, v)?; - } + write_comma_separated_map(f, settings)?; write!(f, " )")?; write!( f, diff --git a/src/query/ast/src/ast/statements/merge_into.rs b/src/query/ast/src/ast/statements/merge_into.rs index 672553bd27350..576a59db64caa 100644 --- a/src/query/ast/src/ast/statements/merge_into.rs +++ b/src/query/ast/src/ast/statements/merge_into.rs @@ -21,6 +21,7 @@ use common_exception::Result; use super::Hint; use crate::ast::write_comma_separated_list; +use crate::ast::write_comma_separated_map; use crate::ast::write_dot_separated_list; use crate::ast::Expr; use crate::ast::Identifier; @@ -257,9 +258,7 @@ impl Display for MergeSource { start: _, } => { write!(f, " FILE_FORMAT = (")?; - for (k, v) in settings.iter() { - write!(f, " {} = '{}'", k, v)?; - } + write_comma_separated_map(f, settings)?; write!(f, " )")?; write!( f, diff --git a/src/query/ast/src/ast/statements/stage.rs b/src/query/ast/src/ast/statements/stage.rs index fe7a4c55e1214..f9fed35e3fd4f 100644 --- a/src/query/ast/src/ast/statements/stage.rs +++ b/src/query/ast/src/ast/statements/stage.rs @@ -17,6 +17,9 @@ use std::default::Default; use std::fmt::Display; use std::fmt::Formatter; +use crate::ast::write_comma_separated_arrow_map; +use crate::ast::write_comma_separated_map; +use crate::ast::write_comma_separated_quoted_list; use crate::ast::UriLocation; #[derive(Debug, Clone, PartialEq, Eq)] @@ -48,9 +51,7 @@ impl Display for CreateStageStmt { if !self.file_format_options.is_empty() { write!(f, " FILE_FORMAT = (")?; - for (k, v) in self.file_format_options.iter() { - write!(f, " {} = '{}'", k, v)?; - } + write_comma_separated_map(f, &self.file_format_options)?; write!(f, " )")?; } @@ -118,35 +119,24 @@ impl Display for SelectStageOptions { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, " (")?; - let mut output: Vec = vec![]; - if let Some(files) = self.files.clone() { - let files = files - .iter() - .map(|x| format!("'{}'", x)) - .collect::>(); - let files = files.join(","); - let files = format!("FILES => ({})", files); - output.push(files); + if let Some(files) = self.files.as_ref() { + write!(f, " FILES => (")?; + write_comma_separated_quoted_list(f, files)?; + write!(f, "),")?; } - if let Some(file_format) = self.file_format.clone() { - let file_format = format!("FILE_FORMAT => '{}'", file_format); - output.push(file_format); + if let Some(file_format) = self.file_format.as_ref() { + write!(f, " FILE_FORMAT => '{}',", file_format)?; } - if let Some(pattern) = self.pattern.clone() { - let pattern = format!("PATTERN => '{}'", pattern); - output.push(pattern); + if let Some(pattern) = self.pattern.as_ref() { + write!(f, " PATTERN => '{}',", pattern)?; } - if !self.connection.is_empty() { - for (k, v) in self.connection.iter() { - output.push(format!(" {} => '{}'", k, v)); - } - } + write_comma_separated_arrow_map(f, &self.connection)?; + + write!(f, " )")?; - let output = output.join(","); - write!(f, "{output})")?; Ok(()) } } diff --git a/src/query/ast/src/ast/statements/table.rs b/src/query/ast/src/ast/statements/table.rs index 1dcdca0cbb465..ae90ee03f5e7f 100644 --- a/src/query/ast/src/ast/statements/table.rs +++ b/src/query/ast/src/ast/statements/table.rs @@ -19,8 +19,8 @@ use std::format; use crate::ast::statements::show::ShowLimit; use crate::ast::write_comma_separated_list; +use crate::ast::write_comma_separated_map; use crate::ast::write_dot_separated_list; -use crate::ast::write_space_separated_map; use crate::ast::Expr; use crate::ast::Identifier; use crate::ast::Query; @@ -167,7 +167,7 @@ impl Display for CreateTableStmt { } // Format table options - write_space_separated_map(f, self.table_options.iter())?; + write_comma_separated_map(f, &self.table_options)?; if let Some(as_query) = &self.as_query { write!(f, " AS {as_query}")?; } @@ -354,34 +354,34 @@ impl Display for AlterTableAction { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { match self { AlterTableAction::SetOptions { set_options } => { - write!(f, "SET OPTIONS: ").expect("Set Options Write Error "); - write_space_separated_map(f, set_options.iter()) + write!(f, "SET OPTIONS (")?; + write_comma_separated_map(f, set_options)?; + write!(f, ")")?; } AlterTableAction::RenameTable { new_table } => { - write!(f, "RENAME TO {new_table}") + write!(f, "RENAME TO {new_table}")?; } AlterTableAction::RenameColumn { old_column, new_column, } => { - write!(f, "RENAME COLUMN {old_column} TO {new_column}") + write!(f, "RENAME COLUMN {old_column} TO {new_column}")?; } AlterTableAction::AddColumn { column, option } => { write!(f, "ADD COLUMN {column}{option}")?; - Ok(()) } AlterTableAction::ModifyColumn { action } => { - write!(f, "MODIFY COLUMN {action}") + write!(f, "MODIFY COLUMN {action}")?; } AlterTableAction::DropColumn { column } => { - write!(f, "DROP COLUMN {column}") + write!(f, "DROP COLUMN {column}")?; } AlterTableAction::AlterTableClusterKey { cluster_by } => { write!(f, "CLUSTER BY ")?; - write_comma_separated_list(f, cluster_by) + write_comma_separated_list(f, cluster_by)?; } AlterTableAction::DropTableClusterKey => { - write!(f, "DROP CLUSTER KEY") + write!(f, "DROP CLUSTER KEY")?; } AlterTableAction::ReclusterTable { is_final, @@ -398,13 +398,12 @@ impl Display for AlterTableAction { if let Some(limit) = limit { write!(f, " LIMIT {limit}")?; } - Ok(()) } AlterTableAction::RevertTo { point } => { write!(f, "REVERT TO {}", point)?; - Ok(()) } - } + }; + Ok(()) } } diff --git a/src/query/ast/src/parser/stage.rs b/src/query/ast/src/parser/stage.rs index e0ed41ca135dc..cd3e46a327617 100644 --- a/src/query/ast/src/parser/stage.rs +++ b/src/query/ast/src/parser/stage.rs @@ -31,9 +31,10 @@ pub fn parameter_to_string(i: Input) -> IResult { let ident_to_string = |i| map_res(ident, |ident| Ok(ident.name))(i); let u64_to_string = |i| map(literal_u64, |v| v.to_string())(i); - map( - rule! { ( #literal_string | #ident_to_string | #u64_to_string ) }, - |parameter| parameter, + rule! ( + #literal_string + | #ident_to_string + | #u64_to_string )(i) } @@ -41,14 +42,14 @@ fn connection_opt(sep: &'static str) -> impl FnMut(Input) -> IResult<(String, St move |i| { let string_options = map( rule! { - ( #ident ) ~ #match_text(sep) ~ #literal_string + #ident ~ #match_text(sep) ~ #literal_string }, |(k, _, v)| (k.to_string().to_lowercase(), v), ); let bool_options = map( rule! { - (ENABLE_VIRTUAL_HOST_STYLE) ~ #match_text(sep) ~ #literal_bool + ENABLE_VIRTUAL_HOST_STYLE ~ #match_text(sep) ~ #literal_bool }, |(k, _, v)| (k.text().to_string().to_lowercase(), v.to_string()), ); @@ -59,22 +60,25 @@ fn connection_opt(sep: &'static str) -> impl FnMut(Input) -> IResult<(String, St pub fn connection_options(i: Input) -> IResult> { let connection_opt = connection_opt("="); - map(rule! { "(" ~ (#connection_opt)* ~ ")"}, |(_, opts, _)| { - BTreeMap::from_iter(opts.iter().map(|(k, v)| (k.to_lowercase(), v.clone()))) - })(i) + map( + rule! { "(" ~ ( #connection_opt ~ ","? )* ~ ^")" }, + |(_, opts, _)| { + BTreeMap::from_iter(opts.iter().map(|((k, v), _)| (k.to_lowercase(), v.clone()))) + }, + )(i) } pub fn format_options(i: Input) -> IResult> { let option_type = map( rule! { - TYPE ~ "=" ~ (TSV | CSV | NDJSON | PARQUET | JSON | XML) + TYPE ~ "=" ~ ( TSV | CSV | NDJSON | PARQUET | JSON | XML ) }, |(_, _, v)| ("type".to_string(), v.text().to_string()), ); let option_compression = map( rule! { - COMPRESSION ~ "=" ~ (AUTO | NONE | GZIP | BZ2 | BROTLI | ZSTD | DEFLATE | RAWDEFLATE | XZ) + COMPRESSION ~ "=" ~ ( AUTO | NONE | GZIP | BZ2 | BROTLI | ZSTD | DEFLATE | RAWDEFLATE | XZ ) }, |(_, _, v)| ("COMPRESSION".to_string(), v.text().to_string()), ); @@ -90,14 +94,14 @@ pub fn format_options(i: Input) -> IResult> { | NAN_DISPLAY | NULL_DISPLAY | ESCAPE - | ROW_TAG) ~ "=" ~ #literal_string + | ROW_TAG) ~ ^"=" ~ ^#literal_string }, |(k, _, v)| (k.text().to_string(), v), ); let int_options = map( rule! { - SKIP_HEADER ~ "=" ~ #literal_u64 + SKIP_HEADER ~ ^"=" ~ ^#literal_u64 }, |(k, _, v)| (k.text().to_string(), v.to_string()), ); @@ -111,7 +115,12 @@ pub fn format_options(i: Input) -> IResult> { let none_options = map( rule! { - (RECORD_DELIMITER | FIELD_DELIMITER | QUOTE | SKIP_HEADER | NON_DISPLAY | ESCAPE ) ~ "=" ~ NONE + (RECORD_DELIMITER + | FIELD_DELIMITER + | QUOTE + | SKIP_HEADER + | NON_DISPLAY + | ESCAPE ) ~ ^"=" ~ ^NONE }, |(k, _, v)| (k.text().to_string(), v.text().to_string()), ); @@ -133,12 +142,12 @@ pub fn file_format_clause(i: Input) -> IResult> { pub fn options(i: Input) -> IResult> { map( rule! { - "(" ~ ( #ident ~ ^"=" ~ ^#parameter_to_string )* ~ ^")" + "(" ~ ( #ident ~ ^"=" ~ ^#parameter_to_string ~ ","? )* ~ ^")" }, |(_, opts, _)| { BTreeMap::from_iter( opts.iter() - .map(|(k, _, v)| (k.name.to_lowercase(), v.clone())), + .map(|(k, _, v, _)| (k.name.to_lowercase(), v.clone())), ) }, )(i) @@ -206,15 +215,15 @@ pub fn string_location(i: Input) -> IResult { pub fn select_stage_option(i: Input) -> IResult { alt(( map( - rule! { FILES ~ "=>" ~ "(" ~ #comma_separated_list0(literal_string) ~ ")" }, + rule! { FILES ~ ^"=>" ~ ^"(" ~ ^#comma_separated_list0(literal_string) ~ ^")" }, |(_, _, _, files, _)| SelectStageOption::Files(files), ), map( - rule! { PATTERN ~ "=>" ~ #literal_string }, + rule! { PATTERN ~ ^"=>" ~ ^#literal_string }, |(_, _, pattern)| SelectStageOption::Pattern(pattern), ), map( - rule! { FILE_FORMAT ~ "=>" ~ #literal_string }, + rule! { FILE_FORMAT ~ ^"=>" ~ ^#literal_string }, |(_, _, file_format)| SelectStageOption::FileFormat(file_format), ), map(connection_opt("=>"), SelectStageOption::Connection), diff --git a/src/query/ast/src/parser/statement.rs b/src/query/ast/src/parser/statement.rs index 1550b916ae444..82062c5cd3986 100644 --- a/src/query/ast/src/parser/statement.rs +++ b/src/query/ast/src/parser/statement.rs @@ -301,7 +301,7 @@ pub fn statement(i: Input) -> IResult { CREATE ~ CATALOG ~ ( IF ~ ^NOT ~ ^EXISTS )? ~ #ident ~ TYPE ~ "=" ~ #catalog_type - ~ CONNECTION ~ "=" ~ #options + ~ CONNECTION ~ "=" ~ #connection_options }, |(_, _, opt_if_not_exists, catalog, _, _, ty, _, _, options)| { Statement::CreateCatalog(CreateCatalogStmt { @@ -1014,12 +1014,12 @@ pub fn statement(i: Input) -> IResult { rule! { CREATE ~ STAGE ~ ( IF ~ ^NOT ~ ^EXISTS )? ~ ( #stage_name ) - ~ ( URL ~ ^"=" ~ ^#uri_location)? + ~ ( URL ~ ^"=" ~ ^#uri_location )? ~ ( #file_format_clause )? - ~ ( ON_ERROR ~ ^"=" ~ ^#ident)? - ~ ( SIZE_LIMIT ~ ^"=" ~ ^#literal_u64)? - ~ ( VALIDATION_MODE ~ ^"=" ~ ^#ident)? - ~ ( (COMMENT | COMMENTS) ~ ^"=" ~ ^#literal_string)? + ~ ( ON_ERROR ~ ^"=" ~ ^#ident )? + ~ ( SIZE_LIMIT ~ ^"=" ~ ^#literal_u64 )? + ~ ( VALIDATION_MODE ~ ^"=" ~ ^#ident )? + ~ ( (COMMENT | COMMENTS) ~ ^"=" ~ ^#literal_string )? }, |( _, @@ -1093,7 +1093,7 @@ pub fn statement(i: Input) -> IResult { ~ #hint? ~ INTO ~ #copy_unit ~ FROM ~ #copy_unit - ~ ( #copy_option ~ ","? )* + ~ #copy_option* }, |(_, opt_hints, _, dst, _, src, opts)| { let mut copy_stmt = CopyStmt { @@ -1114,7 +1114,7 @@ pub fn statement(i: Input) -> IResult { disable_variant_check: Default::default(), on_error: "abort".to_string(), }; - for (opt, _) in opts { + for opt in opts { copy_stmt.apply_option(opt); } Statement::Copy(copy_stmt) diff --git a/src/query/ast/tests/it/parser.rs b/src/query/ast/tests/it/parser.rs index 0182b5df19fed..230fb54c39669 100644 --- a/src/query/ast/tests/it/parser.rs +++ b/src/query/ast/tests/it/parser.rs @@ -169,7 +169,7 @@ fn test_statement() { r#"insert into table t select * from t2;"#, r#"select parse_json('{"k1": [0, 1, 2]}').k1[0];"#, r#"CREATE STAGE ~"#, - r#"CREATE STAGE IF NOT EXISTS test_stage url='s3://load/files/' credentials=(aws_key_id='1a2b3c' aws_secret_key='4x5y6z') file_format=(type = CSV compression = GZIP record_delimiter=',')"#, + r#"CREATE STAGE IF NOT EXISTS test_stage url='s3://load/files/' credentials=(aws_key_id='1a2b3c', aws_secret_key='4x5y6z') file_format=(type = CSV, compression = GZIP record_delimiter=',')"#, r#"CREATE STAGE IF NOT EXISTS test_stage url='azblob://load/files/' connection=(account_name='1a2b3c' account_key='4x5y6z') file_format=(type = CSV compression = GZIP record_delimiter=',')"#, r#"DROP STAGE abc"#, r#"DROP STAGE ~"#, @@ -256,8 +256,8 @@ fn test_statement() { field_delimiter = ',' record_delimiter = '\n' skip_header = 1 - ), - size_limit=10,;"#, + ) + size_limit=10;"#, r#"COPY INTO mytable FROM 's3://mybucket/data.csv' FILE_FORMAT = ( @@ -266,7 +266,7 @@ fn test_statement() { record_delimiter = '\n' skip_header = 1 ) - size_limit=10, + size_limit=10 max_files=10;"#, r#"COPY INTO mytable FROM 's3://mybucket/data.csv' @@ -335,21 +335,21 @@ fn test_statement() { field_delimiter = ',' record_delimiter = '\n' skip_header = 1 - ), + ) size_limit=10;"#, r#"COPY INTO mytable FROM 's3://mybucket/data.csv' CREDENTIALS = ( AWS_KEY_ID = 'access_key' AWS_SECRET_KEY = 'secret_key' - ), + ) FILE_FORMAT = ( type = CSV field_delimiter = ',' record_delimiter = '\n' skip_header = 1 - ), - size_limit=10,;"#, + ) + size_limit=10;"#, r#"COPY INTO mytable FROM @external_stage/path/to/file.csv FILE_FORMAT = ( @@ -388,6 +388,14 @@ fn test_statement() { size_limit=10 disable_variant_check=true;"#, r#"copy into t1 from "" FILE_FORMAT = (TYPE = TSV, COMPRESSION = GZIP)"#, + r#"COPY INTO books FROM 's3://databend/books.csv' + CONNECTION = ( + ENDPOINT_URL = 'http://localhost:9000/', + ACCESS_KEY_ID = 'ROOTUSER', + SECRET_ACCESS_KEY = 'CHANGEME123', + region = 'us-west-2' + ) + FILE_FORMAT = (type = CSV);"#, // We used to support COPY FROM a quoted at string // r#"COPY INTO mytable // FROM '@external_stage/path/to/file.csv' diff --git a/src/query/ast/tests/it/testdata/statement-error.txt b/src/query/ast/tests/it/testdata/statement-error.txt index d3a5bfe46ea84..9c217ca8c7903 100644 --- a/src/query/ast/tests/it/testdata/statement-error.txt +++ b/src/query/ast/tests/it/testdata/statement-error.txt @@ -570,7 +570,7 @@ error: --> SQL:1:41 | 1 | copy into t1 from "" FILE_FORMAT = (TYPE - | ---- ^ expected `)` or `=` + | ---- ^ expected `=` | | | while parsing `COPY INTO { internalStage | externalStage | externalLocation | [.] } @@ -589,7 +589,7 @@ error: --> SQL:1:43 | 1 | copy into t1 from "" FILE_FORMAT = (TYPE = - | ---- ^ expected `)`, `TSV`, `CSV`, `NDJSON`, `PARQUET`, `JSON`, `XML`, or + | ---- ^ expected , `TSV`, `CSV`, `NDJSON`, `PARQUET`, `JSON`, or `XML` | | | while parsing `COPY INTO { internalStage | externalStage | externalLocation | [.] } @@ -608,7 +608,7 @@ error: --> SQL:1:43 | 1 | copy into t1 from "" FILE_FORMAT = (TYPE = - | ---- ^ expected `)`, `TSV`, `CSV`, `NDJSON`, `PARQUET`, `JSON`, `XML`, or + | ---- ^ expected , `TSV`, `CSV`, `NDJSON`, `PARQUET`, `JSON`, or `XML` | | | while parsing `COPY INTO { internalStage | externalStage | externalLocation | [.] } diff --git a/src/query/ast/tests/it/testdata/statement.txt b/src/query/ast/tests/it/testdata/statement.txt index 8c97eef733f53..c67cc9037b01d 100644 --- a/src/query/ast/tests/it/testdata/statement.txt +++ b/src/query/ast/tests/it/testdata/statement.txt @@ -1569,7 +1569,7 @@ UseDatabase { ---------- Input ---------- create catalog ctl type=hive connection=(url='' thrift_protocol='binary'); ---------- Output --------- -CREATE CATALOG ctl TYPE='HIVE' CONNECTION = ( thrift_protocol='binary' url='' ) +CREATE CATALOG ctl TYPE='HIVE' CONNECTION = ( thrift_protocol='binary', url='' ) ---------- AST ------------ CreateCatalog( CreateCatalogStmt { @@ -6507,7 +6507,7 @@ Query( ---------- Input ---------- select * from @foo (pattern=>'[.]*parquet' file_format=>'tsv'); ---------- Output --------- -SELECT * FROM @foo (FILE_FORMAT => 'tsv',PATTERN => '[.]*parquet') +SELECT * FROM @foo ( FILE_FORMAT => 'tsv', PATTERN => '[.]*parquet', ) ---------- AST ------------ Query( Query { @@ -7012,9 +7012,9 @@ CreateStage( ---------- Input ---------- -CREATE STAGE IF NOT EXISTS test_stage url='s3://load/files/' credentials=(aws_key_id='1a2b3c' aws_secret_key='4x5y6z') file_format=(type = CSV compression = GZIP record_delimiter=',') +CREATE STAGE IF NOT EXISTS test_stage url='s3://load/files/' credentials=(aws_key_id='1a2b3c', aws_secret_key='4x5y6z') file_format=(type = CSV, compression = GZIP record_delimiter=',') ---------- Output --------- -CREATE STAGE IF NOT EXISTS test_stage URL = 's3://load/files/' CONNECTION = ( aws_key_id='1a2b3c' aws_secret_key='4x5y6z' ) FILE_FORMAT = ( compression = 'GZIP' record_delimiter = ',' type = 'CSV' ) +CREATE STAGE IF NOT EXISTS test_stage URL = 's3://load/files/' CONNECTION = ( aws_key_id='1a2b3c', aws_secret_key='4x5y6z' ) FILE_FORMAT = (compression='GZIP', record_delimiter=',', type='CSV' ) ---------- AST ------------ CreateStage( CreateStageStmt { @@ -7051,7 +7051,7 @@ CreateStage( ---------- Input ---------- CREATE STAGE IF NOT EXISTS test_stage url='azblob://load/files/' connection=(account_name='1a2b3c' account_key='4x5y6z') file_format=(type = CSV compression = GZIP record_delimiter=',') ---------- Output --------- -CREATE STAGE IF NOT EXISTS test_stage URL = 'azblob://load/files/' CONNECTION = ( account_key='4x5y6z' account_name='1a2b3c' ) FILE_FORMAT = ( compression = 'GZIP' record_delimiter = ',' type = 'CSV' ) +CREATE STAGE IF NOT EXISTS test_stage URL = 'azblob://load/files/' CONNECTION = ( account_key='4x5y6z', account_name='1a2b3c' ) FILE_FORMAT = (compression='GZIP', record_delimiter=',', type='CSV' ) ---------- AST ------------ CreateStage( CreateStageStmt { @@ -8098,7 +8098,7 @@ AlterTable( ---------- Input ---------- ALTER TABLE t SET OPTIONS(SNAPSHOT_LOCATION='1/7/_ss/101fd790dbbe4238a31a8f2e2f856179_v4.mpk',block_per_segment = 500); ---------- Output --------- -ALTER TABLE t SET OPTIONS: block_per_segment='500' snapshot_location='1/7/_ss/101fd790dbbe4238a31a8f2e2f856179_v4.mpk' +ALTER TABLE t SET OPTIONS (block_per_segment='500', snapshot_location='1/7/_ss/101fd790dbbe4238a31a8f2e2f856179_v4.mpk') ---------- AST ------------ AlterTable( AlterTableStmt { @@ -9226,11 +9226,11 @@ COPY INTO mytable field_delimiter = ',' record_delimiter = '\n' skip_header = 1 - ), - size_limit=10,; + ) + size_limit=10; ---------- Output --------- -COPY INTO mytable FROM @~/mybucket/data.csv FILE_FORMAT = ( field_delimiter = ',' record_delimiter = ' -' skip_header = '1' type = 'CSV' ) SIZE_LIMIT = 10 SINGLE = false PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = 'abort' +COPY INTO mytable FROM @~/mybucket/data.csv FILE_FORMAT = (field_delimiter=',', record_delimiter=' +', skip_header='1', type='CSV') SIZE_LIMIT = 10 SINGLE = false PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = 'abort' ---------- AST ------------ Copy( CopyStmt { @@ -9283,11 +9283,11 @@ COPY INTO mytable record_delimiter = '\n' skip_header = 1 ) - size_limit=10, + size_limit=10 max_files=10; ---------- Output --------- -COPY INTO mytable FROM 's3://mybucket/data.csv' FILE_FORMAT = ( field_delimiter = ',' record_delimiter = ' -' skip_header = '1' type = 'CSV' ) SIZE_LIMIT = 10 MAX_FILES = 10 SINGLE = false PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = 'abort' +COPY INTO mytable FROM 's3://mybucket/data.csv' FILE_FORMAT = (field_delimiter=',', record_delimiter=' +', skip_header='1', type='CSV') SIZE_LIMIT = 10 MAX_FILES = 10 SINGLE = false PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = 'abort' ---------- AST ------------ Copy( CopyStmt { @@ -9352,8 +9352,8 @@ COPY INTO mytable size_limit=10 max_files=3000; ---------- Output --------- -COPY INTO mytable FROM 's3://mybucket/data.csv' FILE_FORMAT = ( field_delimiter = ',' record_delimiter = ' -' skip_header = '1' type = 'CSV' ) SIZE_LIMIT = 10 MAX_FILES = 2000 SINGLE = false PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = 'abort' +COPY INTO mytable FROM 's3://mybucket/data.csv' FILE_FORMAT = (field_delimiter=',', record_delimiter=' +', skip_header='1', type='CSV') SIZE_LIMIT = 10 MAX_FILES = 2000 SINGLE = false PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = 'abort' ---------- AST ------------ Copy( CopyStmt { @@ -9420,8 +9420,8 @@ COPY INTO mytable ) size_limit=10; ---------- Output --------- -COPY INTO mytable FROM 's3://mybucket/data.csv' CONNECTION = ( endpoint_url='http://127.0.0.1:9900' ) FILE_FORMAT = ( field_delimiter = ',' record_delimiter = ' -' skip_header = '1' type = 'CSV' ) SIZE_LIMIT = 10 SINGLE = false PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = 'abort' +COPY INTO mytable FROM 's3://mybucket/data.csv' CONNECTION = ( endpoint_url='http://127.0.0.1:9900' ) FILE_FORMAT = (field_delimiter=',', record_delimiter=' +', skip_header='1', type='CSV') SIZE_LIMIT = 10 SINGLE = false PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = 'abort' ---------- AST ------------ Copy( CopyStmt { @@ -9490,8 +9490,8 @@ COPY INTO mytable skip_header = 1 ); ---------- Output --------- -COPY INTO mytable FROM 's3://mybucket/data.csv' CONNECTION = ( endpoint_url='http://127.0.0.1:9900' ) FILE_FORMAT = ( field_delimiter = ',' record_delimiter = ' -' skip_header = '1' type = 'CSV' ) SIZE_LIMIT = 10 SINGLE = false PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = 'abort' +COPY INTO mytable FROM 's3://mybucket/data.csv' CONNECTION = ( endpoint_url='http://127.0.0.1:9900' ) FILE_FORMAT = (field_delimiter=',', record_delimiter=' +', skip_header='1', type='CSV') SIZE_LIMIT = 10 SINGLE = false PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = 'abort' ---------- AST ------------ Copy( CopyStmt { @@ -9662,8 +9662,8 @@ COPY INTO mytable ) size_limit=10; ---------- Output --------- -COPY INTO mytable FROM @my_stage FILE_FORMAT = ( error_on_column_count_mismatch = 'false' field_delimiter = ',' record_delimiter = ' -' skip_header = '1' type = 'CSV' ) SIZE_LIMIT = 10 SINGLE = false PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = 'abort' +COPY INTO mytable FROM @my_stage FILE_FORMAT = (error_on_column_count_mismatch='false', field_delimiter=',', record_delimiter=' +', skip_header='1', type='CSV') SIZE_LIMIT = 10 SINGLE = false PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = 'abort' ---------- AST ------------ Copy( CopyStmt { @@ -9719,8 +9719,8 @@ COPY INTO 's3://mybucket/data.csv' ) size_limit=10; ---------- Output --------- -COPY INTO 's3://mybucket/data.csv' FROM mytable FILE_FORMAT = ( field_delimiter = ',' record_delimiter = ' -' skip_header = '1' type = 'CSV' ) SIZE_LIMIT = 10 SINGLE = false PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = 'abort' +COPY INTO 's3://mybucket/data.csv' FROM mytable FILE_FORMAT = (field_delimiter=',', record_delimiter=' +', skip_header='1', type='CSV') SIZE_LIMIT = 10 SINGLE = false PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = 'abort' ---------- AST ------------ Copy( CopyStmt { @@ -9825,11 +9825,11 @@ COPY INTO @my_stage field_delimiter = ',' record_delimiter = '\n' skip_header = 1 - ), + ) size_limit=10; ---------- Output --------- -COPY INTO @my_stage FROM mytable FILE_FORMAT = ( field_delimiter = ',' record_delimiter = ' -' skip_header = '1' type = 'CSV' ) SIZE_LIMIT = 10 SINGLE = false PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = 'abort' +COPY INTO @my_stage FROM mytable FILE_FORMAT = (field_delimiter=',', record_delimiter=' +', skip_header='1', type='CSV') SIZE_LIMIT = 10 SINGLE = false PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = 'abort' ---------- AST ------------ Copy( CopyStmt { @@ -9879,17 +9879,17 @@ COPY INTO mytable CREDENTIALS = ( AWS_KEY_ID = 'access_key' AWS_SECRET_KEY = 'secret_key' - ), + ) FILE_FORMAT = ( type = CSV field_delimiter = ',' record_delimiter = '\n' skip_header = 1 - ), - size_limit=10,; + ) + size_limit=10; ---------- Output --------- -COPY INTO mytable FROM 's3://mybucket/data.csv' CONNECTION = ( aws_key_id='access_key' aws_secret_key='secret_key' ) FILE_FORMAT = ( field_delimiter = ',' record_delimiter = ' -' skip_header = '1' type = 'CSV' ) SIZE_LIMIT = 10 SINGLE = false PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = 'abort' +COPY INTO mytable FROM 's3://mybucket/data.csv' CONNECTION = ( aws_key_id='access_key', aws_secret_key='secret_key' ) FILE_FORMAT = (field_delimiter=',', record_delimiter=' +', skip_header='1', type='CSV') SIZE_LIMIT = 10 SINGLE = false PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = 'abort' ---------- AST ------------ Copy( CopyStmt { @@ -9956,8 +9956,8 @@ COPY INTO mytable ) size_limit=10; ---------- Output --------- -COPY INTO mytable FROM @external_stage/path/to/file.csv FILE_FORMAT = ( field_delimiter = ',' record_delimiter = ' -' skip_header = '1' type = 'CSV' ) SIZE_LIMIT = 10 SINGLE = false PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = 'abort' +COPY INTO mytable FROM @external_stage/path/to/file.csv FILE_FORMAT = (field_delimiter=',', record_delimiter=' +', skip_header='1', type='CSV') SIZE_LIMIT = 10 SINGLE = false PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = 'abort' ---------- AST ------------ Copy( CopyStmt { @@ -10012,8 +10012,8 @@ COPY INTO mytable ) size_limit=10; ---------- Output --------- -COPY INTO mytable FROM @external_stage/path/to/dir/ FILE_FORMAT = ( field_delimiter = ',' record_delimiter = ' -' skip_header = '1' type = 'CSV' ) SIZE_LIMIT = 10 SINGLE = false PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = 'abort' +COPY INTO mytable FROM @external_stage/path/to/dir/ FILE_FORMAT = (field_delimiter=',', record_delimiter=' +', skip_header='1', type='CSV') SIZE_LIMIT = 10 SINGLE = false PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = 'abort' ---------- AST ------------ Copy( CopyStmt { @@ -10068,8 +10068,8 @@ COPY INTO mytable ) force=true; ---------- Output --------- -COPY INTO mytable FROM @external_stage/path/to/file.csv FILE_FORMAT = ( field_delimiter = ',' record_delimiter = ' -' skip_header = '1' type = 'CSV' ) SINGLE = false PURGE = false FORCE = true DISABLE_VARIANT_CHECK = false ON_ERROR = 'abort' +COPY INTO mytable FROM @external_stage/path/to/file.csv FILE_FORMAT = (field_delimiter=',', record_delimiter=' +', skip_header='1', type='CSV') SINGLE = false PURGE = false FORCE = true DISABLE_VARIANT_CHECK = false ON_ERROR = 'abort' ---------- AST ------------ Copy( CopyStmt { @@ -10125,8 +10125,8 @@ COPY INTO mytable size_limit=10 disable_variant_check=true; ---------- Output --------- -COPY INTO mytable FROM 'fs:///path/to/data.csv' FILE_FORMAT = ( field_delimiter = ',' record_delimiter = ' -' skip_header = '1' type = 'CSV' ) SIZE_LIMIT = 10 SINGLE = false PURGE = false FORCE = false DISABLE_VARIANT_CHECK = true ON_ERROR = 'abort' +COPY INTO mytable FROM 'fs:///path/to/data.csv' FILE_FORMAT = (field_delimiter=',', record_delimiter=' +', skip_header='1', type='CSV') SIZE_LIMIT = 10 SINGLE = false PURGE = false FORCE = false DISABLE_VARIANT_CHECK = true ON_ERROR = 'abort' ---------- AST ------------ Copy( CopyStmt { @@ -10182,7 +10182,7 @@ Copy( ---------- Input ---------- copy into t1 from "" FILE_FORMAT = (TYPE = TSV, COMPRESSION = GZIP) ---------- Output --------- -COPY INTO t1 FROM "" FILE_FORMAT = ( compression = 'GZIP' type = 'TSV' ) SINGLE = false PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = 'abort' +COPY INTO t1 FROM "" FILE_FORMAT = (compression='GZIP', type='TSV') SINGLE = false PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = 'abort' ---------- AST ------------ Copy( CopyStmt { @@ -10233,6 +10233,71 @@ Copy( ) +---------- Input ---------- +COPY INTO books FROM 's3://databend/books.csv' + CONNECTION = ( + ENDPOINT_URL = 'http://localhost:9000/', + ACCESS_KEY_ID = 'ROOTUSER', + SECRET_ACCESS_KEY = 'CHANGEME123', + region = 'us-west-2' + ) + FILE_FORMAT = (type = CSV); +---------- Output --------- +COPY INTO books FROM 's3://databend/books.csv' CONNECTION = ( access_key_id='ROOTUSER', endpoint_url='http://localhost:9000/', region='us-west-2', secret_access_key='CHANGEME123' ) FILE_FORMAT = (type='CSV') SINGLE = false PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = 'abort' +---------- AST ------------ +Copy( + CopyStmt { + hints: None, + src: Location( + Uri( + UriLocation { + protocol: "s3", + name: "databend", + path: "/books.csv", + part_prefix: "", + connection: Connection { + visited_keys: {}, + conns: { + "access_key_id": "ROOTUSER", + "endpoint_url": "http://localhost:9000/", + "region": "us-west-2", + "secret_access_key": "CHANGEME123", + }, + }, + }, + ), + ), + dst: Table { + catalog: None, + database: None, + table: Identifier { + name: "books", + quote: None, + span: Some( + 10..15, + ), + }, + columns: None, + }, + files: None, + pattern: None, + file_format: { + "type": "CSV", + }, + validation_mode: "", + size_limit: 0, + max_files: 0, + max_file_size: 0, + split_size: 0, + single: false, + purge: false, + force: false, + disable_variant_check: false, + on_error: "abort", + }, +) + + ---------- Input ---------- CALL system$test(a) ---------- Output --------- @@ -11189,7 +11254,7 @@ Query( SELECT t.c1 FROM @stage1/dir/file ( file_format => 'PARQUET', FILES => ('file1', 'file2')) t; ---------- Output --------- -SELECT t.c1 FROM @stage1/dir/file (FILES => ('file1','file2'),FILE_FORMAT => 'PARQUET') AS t +SELECT t.c1 FROM @stage1/dir/file ( FILES => ('file1', 'file2'), FILE_FORMAT => 'PARQUET', ) AS t ---------- AST ------------ Query( Query { @@ -11287,7 +11352,7 @@ select table0.c1, table1.c2 from @stage1/dir/file ( FILE_FORMAT => 'parquet', FILES => ('file1', 'file2')) table0 left join table1; ---------- Output --------- -SELECT table0.c1, table1.c2 FROM @stage1/dir/file (FILES => ('file1','file2'),FILE_FORMAT => 'parquet') AS table0 LEFT OUTER JOIN table1 +SELECT table0.c1, table1.c2 FROM @stage1/dir/file ( FILES => ('file1', 'file2'), FILE_FORMAT => 'parquet', ) AS table0 LEFT OUTER JOIN table1 ---------- AST ------------ Query( Query { @@ -11437,7 +11502,7 @@ Query( ---------- Input ---------- SELECT c1 FROM 's3://test/bucket' (ENDPOINT_URL => 'xxx', PATTERN => '*.parquet') t; ---------- Output --------- -SELECT c1 FROM 's3://test/bucket' (PATTERN => '*.parquet', endpoint_url => 'xxx') AS t +SELECT c1 FROM 's3://test/bucket' ( PATTERN => '*.parquet',endpoint_url=>'xxx' ) AS t ---------- AST ------------ Query( Query { diff --git a/tests/sqllogictests/suites/base/05_ddl/05_0019_ddl_create_view b/tests/sqllogictests/suites/base/05_ddl/05_0019_ddl_create_view index ea3dc47b5bb28..5cdc6f1c30b01 100644 --- a/tests/sqllogictests/suites/base/05_ddl/05_0019_ddl_create_view +++ b/tests/sqllogictests/suites/base/05_ddl/05_0019_ddl_create_view @@ -110,7 +110,7 @@ CREATE VIEW test AS SELECT * FROM - 's3://testbucket/admin/data/tuple.parquet' (FILES => ('tuple.parquet','test.parquet'),FILE_FORMAT => 'parquet',PATTERN => '.*.parquet', aws_key_id => 'minioadmin', aws_secret_key => 'minioadmin', endpoint_url => 'http://127.0.0.1:9900/') + 's3://testbucket/admin/data/tuple.parquet' ( FILES => ('tuple.parquet', 'test.parquet'), FILE_FORMAT => 'parquet', PATTERN => '.*.parquet',aws_key_id=>'minioadmin', aws_secret_key=>'minioadmin', endpoint_url=>'http://127.0.0.1:9900/' ) statement ok drop view if exists loop_view1;