feat: CopyIntoLocation support output. (#14315)

* refactor unload. * feat: CopyIntoLocation support output. * update tests. * fix test. * fix test.
databendlabs · Jan 14, 2024 · b9a4726 · b9a4726
1 parent 094e1db
commit b9a4726
Show file tree

Hide file tree

Showing 40 changed files with 673 additions and 142 deletions.
diff --git a/src/meta/app/src/principal/user_stage.rs b/src/meta/app/src/principal/user_stage.rs
@@ -478,10 +478,13 @@ pub struct CopyOptions {
     pub max_files: usize,
     pub split_size: usize,
     pub purge: bool,
-    pub single: bool,
-    pub max_file_size: usize,
     pub disable_variant_check: bool,
     pub return_failed_only: bool,
+
+    // unload only
+    pub max_file_size: usize,
+    pub single: bool,
+    pub detailed_output: bool,
 }
 
 impl CopyOptions {

diff --git a/src/meta/proto-conv/src/stage_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/stage_from_to_protobuf_impl.rs
@@ -135,6 +135,7 @@ impl FromToProto for mt::principal::CopyOptions {
             max_file_size,
             disable_variant_check: p.disable_variant_check,
             return_failed_only: p.return_failed_only,
+            detailed_output: false,
         })
     }
 

diff --git a/src/meta/proto-conv/tests/it/user_proto_conv.rs b/src/meta/proto-conv/tests/it/user_proto_conv.rs
@@ -99,6 +99,7 @@ pub(crate) fn test_fs_stage_info() -> mt::principal::StageInfo {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
 
@@ -142,6 +143,7 @@ pub(crate) fn test_s3_stage_info() -> mt::principal::StageInfo {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -181,6 +183,7 @@ pub(crate) fn test_s3_stage_info_v16() -> mt::principal::StageInfo {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -220,6 +223,7 @@ pub(crate) fn test_s3_stage_info_v14() -> mt::principal::StageInfo {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -255,6 +259,7 @@ pub(crate) fn test_gcs_stage_info() -> mt::principal::StageInfo {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -294,6 +299,7 @@ pub(crate) fn test_oss_stage_info() -> mt::principal::StageInfo {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -328,6 +334,7 @@ pub(crate) fn test_webhdfs_stage_info() -> mt::principal::StageInfo {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -363,6 +370,7 @@ pub(crate) fn test_obs_stage_info() -> mt::principal::StageInfo {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -398,6 +406,7 @@ pub(crate) fn test_cos_stage_info() -> mt::principal::StageInfo {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -847,6 +856,7 @@ pub(crate) fn test_internal_stage_info_v17() -> mt::principal::StageInfo {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -878,6 +888,7 @@ pub(crate) fn test_stage_info_v18() -> mt::principal::StageInfo {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()

diff --git a/src/meta/proto-conv/tests/it/user_stage.rs b/src/meta/proto-conv/tests/it/user_stage.rs
@@ -111,6 +111,7 @@ fn test_user_stage_webhdfs_v30() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -156,6 +157,7 @@ fn test_user_stage_fs_v22() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -200,6 +202,7 @@ fn test_user_stage_fs_v21() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -243,6 +246,7 @@ fn test_user_stage_fs_v20() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -287,6 +291,7 @@ fn test_user_stage_fs_v16() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -343,6 +348,7 @@ fn test_user_stage_s3_v16() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -393,6 +399,7 @@ fn test_user_stage_gcs_v16() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -448,6 +455,7 @@ fn test_user_stage_oss_v16() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -503,6 +511,7 @@ fn test_user_stage_oss_v13() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -558,6 +567,7 @@ fn test_user_stage_s3_v11() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -611,6 +621,7 @@ fn test_user_stage_s3_v8() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -655,6 +666,7 @@ fn test_user_stage_fs_v6() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -709,6 +721,7 @@ fn test_user_stage_s3_v6() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -759,6 +772,7 @@ fn test_user_stage_gcs_v6() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -802,6 +816,7 @@ fn test_user_stage_fs_v4() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -856,6 +871,7 @@ fn test_user_stage_s3_v4() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -905,6 +921,7 @@ fn test_user_stage_gcs_v4() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -956,6 +973,7 @@ fn test_user_stage_s3_v1() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -1002,6 +1020,7 @@ fn test_internal_stage_v17() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -1047,6 +1066,7 @@ fn test_user_stage_v19() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()

diff --git a/src/meta/proto-conv/tests/it/v025_user_stage.rs b/src/meta/proto-conv/tests/it/v025_user_stage.rs
@@ -68,6 +68,7 @@ fn test_decode_v25_user_stage() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         number_of_files: 100,

diff --git a/src/meta/proto-conv/tests/it/v030_user_stage.rs b/src/meta/proto-conv/tests/it/v030_user_stage.rs
@@ -70,6 +70,7 @@ fn test_decode_v30_user_stage() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()

diff --git a/src/meta/proto-conv/tests/it/v031_copy_max_file.rs b/src/meta/proto-conv/tests/it/v031_copy_max_file.rs
@@ -68,6 +68,7 @@ fn test_decode_v31_copy_max_file() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()

diff --git a/src/meta/proto-conv/tests/it/v035_user_stage.rs b/src/meta/proto-conv/tests/it/v035_user_stage.rs
@@ -65,6 +65,7 @@ fn test_decode_v35_user_stage() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: true,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         number_of_files: 100,

diff --git a/src/meta/proto-conv/tests/it/v042_s3_stage_new_field.rs b/src/meta/proto-conv/tests/it/v042_s3_stage_new_field.rs
@@ -69,6 +69,7 @@ fn test_decode_v42_s3_stage_new_field() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: true,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         number_of_files: 100,

diff --git a/src/meta/proto-conv/tests/it/v051_obs_and_cos_storage.rs b/src/meta/proto-conv/tests/it/v051_obs_and_cos_storage.rs
@@ -71,6 +71,7 @@ fn test_decode_v51_obs_stage() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -120,6 +121,7 @@ fn test_decode_v51_cos_stage() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()

diff --git a/src/meta/proto-conv/tests/it/v057_hdfs_storage.rs b/src/meta/proto-conv/tests/it/v057_hdfs_storage.rs
@@ -64,6 +64,7 @@ fn test_decode_v57_hdfs_storage() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: false,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "test".to_string(),
         ..Default::default()

diff --git a/src/meta/proto-conv/tests/it/v060_copy_options.rs b/src/meta/proto-conv/tests/it/v060_copy_options.rs
@@ -42,6 +42,7 @@ fn test_decode_v60_copy_options() -> anyhow::Result<()> {
         max_file_size: 100,
         disable_variant_check: true,
         return_failed_only: true,
+        detailed_output: false,
     };
     common::test_pb_from_to(func_name!(), want())?;
     common::test_load_old(func_name!(), copy_options_v60.as_slice(), 0, want())?;

diff --git a/src/meta/proto-conv/tests/it/v066_stage_create_on.rs b/src/meta/proto-conv/tests/it/v066_stage_create_on.rs
@@ -69,6 +69,7 @@ fn test_decode_v66_stage() -> anyhow::Result<()> {
             max_file_size: 0,
             disable_variant_check: true,
             return_failed_only: false,
+            detailed_output: false,
         },
         comment: "ccc".to_string(),
         number_of_files: 100,

diff --git a/src/query/ast/src/ast/statements/copy.rs b/src/query/ast/src/ast/statements/copy.rs
@@ -216,6 +216,7 @@ pub struct CopyIntoLocationStmt {
     pub file_format: BTreeMap<String, String>,
     pub single: bool,
     pub max_file_size: usize,
+    pub detailed_output: bool,
 }
 
 impl Display for CopyIntoLocationStmt {
@@ -233,7 +234,8 @@ impl Display for CopyIntoLocationStmt {
             write!(f, ")")?;
         }
         write!(f, " SINGLE = {}", self.single)?;
-        write!(f, " MAX_FILE_SIZE= {}", self.max_file_size)?;
+        write!(f, " MAX_FILE_SIZE = {}", self.max_file_size)?;
+        write!(f, " DETAILED_OUTPUT = {}", self.detailed_output)?;
 
         Ok(())
     }
@@ -245,6 +247,7 @@ impl CopyIntoLocationStmt {
             CopyIntoLocationOption::FileFormat(v) => self.file_format = v,
             CopyIntoLocationOption::Single(v) => self.single = v,
             CopyIntoLocationOption::MaxFileSize(v) => self.max_file_size = v,
+            CopyIntoLocationOption::DetailedOutput(v) => self.detailed_output = v,
         }
     }
 }
@@ -491,4 +494,5 @@ pub enum CopyIntoLocationOption {
     FileFormat(BTreeMap<String, String>),
     MaxFileSize(usize),
     Single(bool),
+    DetailedOutput(bool),
 }