Merge branch 'main' into chore-nextest

databendlabs · Oct 8, 2023 · e3324f5 · e3324f5
2 parents 2cfe0ff + a5533ed
commit e3324f5
Show file tree

Hide file tree

Showing 38 changed files with 389 additions and 155 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -572,7 +572,7 @@ jobs:
           tar x -C ./target/release -f ./distro/databend-testsuite-${version}-${target}.tar.gz --strip-components 1 bin/
           chmod +x ./target/release/databend-*
       - name: Run sqlsmith
-        timeout-minutes: 30
+        timeout-minutes: 60
         shell: bash
         run: |
           bash ./scripts/ci/ci-run-sqlsmith-tests.sh

diff --git a/.github/workflows/reuse.linux.hive.yml b/.github/workflows/reuse.linux.hive.yml
@@ -32,7 +32,7 @@ jobs:
           # fetch all tags, metasrv and metaclient need tag as its version.
           fetch-depth: 0
       - uses: ./.github/actions/build_linux
-        timeout-minutes: 30
+        timeout-minutes: 60
         with:
           sha: ${{ github.sha }}
           target: ${{ matrix.arch }}-unknown-linux-${{ matrix.libc }}

diff --git a/.github/workflows/reuse.linux.yml b/.github/workflows/reuse.linux.yml
@@ -27,7 +27,7 @@ jobs:
           # fetch all tags, metasrv and metaclient need tag as its version.
           fetch-depth: 0
       - uses: ./.github/actions/check
-        timeout-minutes: 30
+        timeout-minutes: 60
         with:
           github_token: ${{ github.token }}
 
@@ -101,7 +101,7 @@ jobs:
           # fetch all tags, metasrv and metaclient need tag as its version.
           fetch-depth: 0
       - uses: ./.github/actions/test_unit
-        timeout-minutes: 30
+        timeout-minutes: 60
 
   test_metactl:
     runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]

diff --git a/src/meta/app/src/principal/user_stage.rs b/src/meta/app/src/principal/user_stage.rs
@@ -475,6 +475,7 @@ pub struct CopyOptions {
     pub single: bool,
     pub max_file_size: usize,
     pub disable_variant_check: bool,
+    pub return_failed_only: bool,
 }
 
 impl CopyOptions {
@@ -525,6 +526,15 @@ impl CopyOptions {
                     })?;
                     self.disable_variant_check = disable_variant_check;
                 }
+                "return_failed_only" => {
+                    let return_failed_only = bool::from_str(v).map_err(|_| {
+                        ErrorCode::StrParseError(format!(
+                            "Cannot parse return_failed_only: {} as bool",
+                            v
+                        ))
+                    })?;
+                    self.return_failed_only = return_failed_only;
+                }
                 _ => {
                     if !ignore_unknown {
                         return Err(ErrorCode::BadArguments(format!(

diff --git a/src/meta/proto-conv/src/stage_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/stage_from_to_protobuf_impl.rs
@@ -134,6 +134,7 @@ impl FromToProto for mt::principal::CopyOptions {
             single: p.single,
             max_file_size,
             disable_variant_check: p.disable_variant_check,
+            return_failed_only: p.return_failed_only,
         })
     }
 
@@ -163,6 +164,7 @@ impl FromToProto for mt::principal::CopyOptions {
             single: self.single,
             max_file_size,
             disable_variant_check: self.disable_variant_check,
+            return_failed_only: self.return_failed_only,
         })
     }
 }

diff --git a/src/meta/proto-conv/src/util.rs b/src/meta/proto-conv/src/util.rs
@@ -89,6 +89,7 @@ const META_CHANGE_LOG: &[(u64, &str)] = &[
     (57, "2023-09-05: Add: catalog.proto add hdfs config", ),
     (58, "2023-09-06: Add: udf.proto/UserDefinedFunction", ),
     (59, "2023-08-17: Add: user.proto/CsvFileFormatParams add field `allow_column_count_mismatch`", ),
+    (60, "2023-08-17: Add: user.proto/CopyOptions add field `return_failed_only`", ),
     // Dear developer:
     //      If you're gonna add a new metadata version, you'll have to add a test for it.
     //      You could just copy an existing test file(e.g., `../tests/it/v024_table_meta.rs`)

diff --git a/src/meta/proto-conv/tests/it/main.rs b/src/meta/proto-conv/tests/it/main.rs
@@ -64,3 +64,4 @@ mod v056_least_visible_time;
 mod v057_hdfs_storage;
 mod v058_udf;
 mod v059_csv_format_params;
+mod v060_copy_options;
diff --git a/src/meta/proto-conv/tests/it/user_proto_conv.rs b/src/meta/proto-conv/tests/it/user_proto_conv.rs
@@ -93,6 +93,7 @@ pub(crate) fn test_fs_stage_info() -> mt::principal::StageInfo {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
 
@@ -133,6 +134,7 @@ pub(crate) fn test_s3_stage_info() -> mt::principal::StageInfo {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -170,6 +172,7 @@ pub(crate) fn test_s3_stage_info_v16() -> mt::principal::StageInfo {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -207,6 +210,7 @@ pub(crate) fn test_s3_stage_info_v14() -> mt::principal::StageInfo {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -240,6 +244,7 @@ pub(crate) fn test_gcs_stage_info() -> mt::principal::StageInfo {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -275,6 +280,7 @@ pub(crate) fn test_oss_stage_info() -> mt::principal::StageInfo {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -307,6 +313,7 @@ pub(crate) fn test_webhdfs_stage_info() -> mt::principal::StageInfo {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -340,6 +347,7 @@ pub(crate) fn test_obs_stage_info() -> mt::principal::StageInfo {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -373,6 +381,7 @@ pub(crate) fn test_cos_stage_info() -> mt::principal::StageInfo {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -814,6 +823,7 @@ pub(crate) fn test_internal_stage_info_v17() -> mt::principal::StageInfo {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -843,6 +853,7 @@ pub(crate) fn test_stage_info_v18() -> mt::principal::StageInfo {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()

diff --git a/src/meta/proto-conv/tests/it/user_stage.rs b/src/meta/proto-conv/tests/it/user_stage.rs
@@ -109,6 +109,7 @@ fn test_user_stage_webhdfs_v30() -> anyhow::Result<()> {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -153,6 +154,7 @@ fn test_user_stage_fs_v22() -> anyhow::Result<()> {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -196,6 +198,7 @@ fn test_user_stage_fs_v21() -> anyhow::Result<()> {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -238,6 +241,7 @@ fn test_user_stage_fs_v20() -> anyhow::Result<()> {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -281,6 +285,7 @@ fn test_user_stage_fs_v16() -> anyhow::Result<()> {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -336,6 +341,7 @@ fn test_user_stage_s3_v16() -> anyhow::Result<()> {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -385,6 +391,7 @@ fn test_user_stage_gcs_v16() -> anyhow::Result<()> {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -437,6 +444,7 @@ fn test_user_stage_oss_v16() -> anyhow::Result<()> {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -491,6 +499,7 @@ fn test_user_stage_oss_v13() -> anyhow::Result<()> {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -545,6 +554,7 @@ fn test_user_stage_s3_v11() -> anyhow::Result<()> {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -597,6 +607,7 @@ fn test_user_stage_s3_v8() -> anyhow::Result<()> {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -640,6 +651,7 @@ fn test_user_stage_fs_v6() -> anyhow::Result<()> {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -693,6 +705,7 @@ fn test_user_stage_s3_v6() -> anyhow::Result<()> {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -742,6 +755,7 @@ fn test_user_stage_gcs_v6() -> anyhow::Result<()> {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -784,6 +798,7 @@ fn test_user_stage_fs_v4() -> anyhow::Result<()> {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -837,6 +852,7 @@ fn test_user_stage_s3_v4() -> anyhow::Result<()> {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -885,6 +901,7 @@ fn test_user_stage_gcs_v4() -> anyhow::Result<()> {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -935,6 +952,7 @@ fn test_user_stage_s3_v1() -> anyhow::Result<()> {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -980,6 +998,7 @@ fn test_internal_stage_v17() -> anyhow::Result<()> {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -1024,6 +1043,7 @@ fn test_user_stage_v19() -> anyhow::Result<()> {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()

diff --git a/src/meta/proto-conv/tests/it/v025_user_stage.rs b/src/meta/proto-conv/tests/it/v025_user_stage.rs
@@ -63,6 +63,7 @@ fn test_decode_v25_user_stage() -> anyhow::Result<()> {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         number_of_files: 100,

diff --git a/src/meta/proto-conv/tests/it/v030_user_stage.rs b/src/meta/proto-conv/tests/it/v030_user_stage.rs
@@ -68,6 +68,7 @@ fn test_decode_v30_user_stage() -> anyhow::Result<()> {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()

diff --git a/src/meta/proto-conv/tests/it/v031_copy_max_file.rs b/src/meta/proto-conv/tests/it/v031_copy_max_file.rs
@@ -66,6 +66,7 @@ fn test_decode_v31_copy_max_file() -> anyhow::Result<()> {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()

diff --git a/src/meta/proto-conv/tests/it/v035_user_stage.rs b/src/meta/proto-conv/tests/it/v035_user_stage.rs
@@ -60,6 +60,7 @@ fn test_decode_v35_user_stage() -> anyhow::Result<()> {
             single: false,
             max_file_size: 0,
             disable_variant_check: true,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         number_of_files: 100,

diff --git a/src/meta/proto-conv/tests/it/v042_s3_stage_new_field.rs b/src/meta/proto-conv/tests/it/v042_s3_stage_new_field.rs
@@ -64,6 +64,7 @@ fn test_decode_v42_s3_stage_new_field() -> anyhow::Result<()> {
             single: false,
             max_file_size: 0,
             disable_variant_check: true,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         number_of_files: 100,

diff --git a/src/meta/proto-conv/tests/it/v051_obs_and_cos_storage.rs b/src/meta/proto-conv/tests/it/v051_obs_and_cos_storage.rs
@@ -69,6 +69,7 @@ fn test_decode_v51_obs_stage() -> anyhow::Result<()> {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()
@@ -117,6 +118,7 @@ fn test_decode_v51_cos_stage() -> anyhow::Result<()> {
             single: false,
             max_file_size: 0,
             disable_variant_check: false,
+            return_failed_only: false,
         },
         comment: "test".to_string(),
         ..Default::default()