Merge branch 'main' into new_hashtable_with_blkinfo_for_target_build

src/meta/app/src/schema/database.rs

@@ -22,7 +22,6 @@ use std::ops::Deref;
 use chrono::DateTime;
 use chrono::Utc;
 
-use crate::schema::Ownership;
 use crate::share::ShareNameIdent;
 use crate::share::ShareSpec;
 
@@ -114,8 +113,6 @@ pub struct DatabaseMeta {
     // shared by share_id
     pub shared_by: BTreeSet<u64>,
     pub from_share: Option<ShareNameIdent>,
-
-    pub owner: Option<Ownership>,
 }
 
 impl Default for DatabaseMeta {
@@ -130,7 +127,6 @@ impl Default for DatabaseMeta {
             drop_on: None,
             shared_by: BTreeSet::new(),
             from_share: None,
-            owner: None,
         }
     }
 }

src/meta/app/src/schema/table.rs

@@ -32,7 +32,6 @@ use databend_common_meta_types::MetaId;
 use maplit::hashmap;
 
 use crate::schema::database::DatabaseNameIdent;
-use crate::schema::Ownership;
 use crate::share::ShareNameIdent;
 use crate::share::ShareSpec;
 use crate::share::ShareTableInfoMap;
@@ -239,7 +238,6 @@ pub struct TableMeta {
     // shared by share_id
     pub shared_by: BTreeSet<u64>,
     pub column_mask_policy: Option<BTreeMap<String, String>>,
-    pub owner: Option<Ownership>,
 }
 
 impl TableMeta {
@@ -354,7 +352,6 @@ impl Default for TableMeta {
             statistics: Default::default(),
             shared_by: BTreeSet::new(),
             column_mask_policy: None,
-            owner: None,
         }
     }
 }

src/meta/proto-conv/src/database_from_to_protobuf_impl.rs

@@ -79,10 +79,6 @@ impl FromToProto for mt::DatabaseMeta {
                 Some(from_share) => Some(share::ShareNameIdent::from_pb(from_share)?),
                 None => None,
             },
-            owner: match p.owner {
-                Some(o) => Some(mt::Ownership::from_pb(o)?),
-                None => None,
-            },
         };
         Ok(v)
     }
@@ -106,10 +102,6 @@ impl FromToProto for mt::DatabaseMeta {
                 Some(from_share) => Some(from_share.to_pb()?),
                 None => None,
             },
-            owner: match &self.owner {
-                Some(o) => Some(o.to_pb()?),
-                None => None,
-            },
         };
         Ok(p)
     }

src/meta/proto-conv/src/table_from_to_protobuf_impl.rs

@@ -218,10 +218,6 @@ impl FromToProto for mt::TableMeta {
             } else {
                 Some(p.column_mask_policy)
             },
-            owner: match p.owner {
-                Some(owner) => Some(mt::Ownership::from_pb(owner)?),
-                None => None,
-            },
         };
         Ok(v)
     }
@@ -258,10 +254,6 @@ impl FromToProto for mt::TableMeta {
             statistics: Some(self.statistics.to_pb()?),
             shared_by: Vec::from_iter(self.shared_by.clone()),
             column_mask_policy: self.column_mask_policy.clone().unwrap_or_default(),
-            owner: match self.owner.as_ref() {
-                Some(o) => Some(o.to_pb()?),
-                None => None,
-            },
         };
         Ok(p)
     }

src/meta/proto-conv/src/util.rs

@@ -103,6 +103,7 @@ const META_CHANGE_LOG: &[(u64, &str)] = &[
     (71, "2024-01-02: Add: user.proto/password options", ),
     (72, "2024-01-09: Add: user.proto/CSVFileFormatParams add field `empty_field_as`", ),
     (73, "2024-01-11: Add: config.proto/StorageConfig add HuggingfaceConfig", ),
+    (74, "2024-01-12: Remove: owner in DatabaseMeta and TableMeta", ),
     // Dear developer:
     //      If you're gonna add a new metadata version, you'll have to add a test for it.
     //      You could just copy an existing test file(e.g., `../tests/it/v024_table_meta.rs`)

src/meta/proto-conv/tests/it/main.rs

@@ -77,3 +77,4 @@ mod v070_binary_type;
 mod v071_user_password;
 mod v072_csv_format_params;
 mod v073_huggingface_config;
+mod v074_table_db_meta;

src/meta/proto-conv/tests/it/proto_conv.rs

@@ -59,7 +59,6 @@ fn new_db_meta_share() -> mt::DatabaseMeta {
             tenant: "tenant".to_string(),
             share_name: "share".to_string(),
         }),
-        owner: None,
     }
 }
 
@@ -74,7 +73,6 @@ fn new_db_meta() -> mt::DatabaseMeta {
         drop_on: None,
         shared_by: BTreeSet::from_iter(vec![1]),
         from_share: None,
-        owner: None,
     }
 }
 
@@ -213,7 +211,6 @@ fn new_table_meta() -> mt::TableMeta {
         statistics: Default::default(),
         shared_by: btreeset! {1},
         column_mask_policy: Some(btreemap! {s("a") => s("b")}),
-        owner: None,
     }
 }
 

src/meta/proto-conv/tests/it/v002_database_meta.rs

@@ -52,7 +52,6 @@ fn test_decode_v2_database_meta() -> anyhow::Result<()> {
         drop_on: None,
         shared_by: BTreeSet::from_iter(vec![1]),
         from_share: None,
-        owner: None,
     };
 
     common::test_pb_from_to(func_name!(), want())?;

src/meta/proto-conv/tests/it/v002_table_meta.rs

@@ -143,7 +143,6 @@ fn test_decode_v2_table_meta() -> anyhow::Result<()> {
         statistics: Default::default(),
         shared_by: BTreeSet::new(),
         column_mask_policy: None,
-        owner: None,
     };
 
     common::test_pb_from_to(func_name!(), want())?;

src/meta/proto-conv/tests/it/v005_database_meta.rs

@@ -57,7 +57,6 @@ fn test_decode_v5_database_meta() -> anyhow::Result<()> {
             tenant: "tenant".to_string(),
             share_name: "share".to_string(),
         }),
-        owner: None,
     };
 
     common::test_pb_from_to(func_name!(), want())?;

src/meta/proto-conv/tests/it/v010_table_meta.rs

@@ -145,7 +145,6 @@ fn test_decode_v10_table_meta() -> anyhow::Result<()> {
         statistics: Default::default(),
         shared_by: BTreeSet::new(),
         column_mask_policy: None,
-        owner: None,
     };
 
     common::test_pb_from_to(func_name!(), want())?;

src/meta/proto-conv/tests/it/v012_table_meta.rs

@@ -147,7 +147,6 @@ fn test_decode_v12_table_meta() -> anyhow::Result<()> {
         statistics: Default::default(),
         shared_by: BTreeSet::new(),
         column_mask_policy: None,
-        owner: None,
     };
 
     common::test_pb_from_to(func_name!(), want())?;

src/meta/proto-conv/tests/it/v023_table_meta.rs

@@ -147,7 +147,6 @@ fn test_decode_v23_table_meta() -> anyhow::Result<()> {
         statistics: Default::default(),
         shared_by: BTreeSet::new(),
         column_mask_policy: None,
-        owner: None,
     };
 
     common::test_pb_from_to(func_name!(), want())?;

src/meta/proto-conv/tests/it/v024_table_meta.rs

@@ -147,7 +147,6 @@ fn test_decode_v24_table_meta() -> anyhow::Result<()> {
         statistics: Default::default(),
         shared_by: BTreeSet::new(),
         column_mask_policy: None,
-        owner: None,
     };
 
     common::test_pb_from_to(func_name!(), want())?;

src/meta/proto-conv/tests/it/v033_table_meta.rs

@@ -149,7 +149,6 @@ fn test_decode_v33_table_meta() -> anyhow::Result<()> {
         statistics: Default::default(),
         shared_by: btreeset! {1},
         column_mask_policy: None,
-        owner: None,
     };
 
     common::test_pb_from_to(func_name!(), want())?;

src/meta/proto-conv/tests/it/v040_table_meta.rs

@@ -149,7 +149,6 @@ fn test_decode_v40_table_meta() -> anyhow::Result<()> {
         statistics: Default::default(),
         shared_by: btreeset! {1},
         column_mask_policy: Some(btreemap! {s("a") => s("b")}),
-        owner: None,
     };
 
     common::test_pb_from_to(func_name!(), want())?;

src/meta/proto-conv/tests/it/v044_table_meta.rs

@@ -100,7 +100,6 @@ fn test_decode_v44_table_meta() -> anyhow::Result<()> {
         statistics: Default::default(),
         shared_by: btreeset! {1},
         column_mask_policy: Some(btreemap! {s("a") => s("b")}),
-        owner: None,
     };
 
     common::test_load_old(func_name!(), bytes.as_slice(), 44, want())?;

src/meta/proto-conv/tests/it/v055_table_meta.rs

@@ -1,3 +1,7 @@
+// Copyright 2023 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //     http://www.apache.org/licenses/LICENSE-2.0
@@ -17,7 +21,6 @@ use databend_common_expression as ce;
 use databend_common_expression::types::NumberDataType;
 use databend_common_expression::ComputedExpr;
 use databend_common_meta_app::schema as mt;
-use databend_common_meta_app::schema::Ownership;
 use databend_common_meta_app::share;
 use maplit::btreemap;
 use maplit::btreeset;
@@ -90,10 +93,6 @@ fn test_decode_v55_table_meta() -> anyhow::Result<()> {
         statistics: Default::default(),
         shared_by: btreeset! {1},
         column_mask_policy: Some(btreemap! {s("a") => s("b")}),
-        owner: Some(Ownership {
-            owner_role_name: "role2".to_string(),
-            updated_on: Default::default(),
-        }),
     };
     common::test_pb_from_to(func_name!(), want())?;
     common::test_load_old(func_name!(), bytes.as_slice(), 55, want())?;
@@ -127,10 +126,6 @@ fn test_decode_v51_database_meta() -> anyhow::Result<()> {
             tenant: "tenant".to_string(),
             share_name: "share".to_string(),
         }),
-        owner: Some(Ownership {
-            owner_role_name: "role1".to_string(),
-            updated_on: Default::default(),
-        }),
     };
 
     common::test_pb_from_to(func_name!(), want())?;

src/meta/proto-conv/tests/it/v074_table_db_meta.rs

@@ -0,0 +1,134 @@
+// Copyright 2023 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::BTreeSet;
+use std::sync::Arc;
+
+use chrono::TimeZone;
+use chrono::Utc;
+use databend_common_expression as ce;
+use databend_common_expression::types::NumberDataType;
+use databend_common_expression::ComputedExpr;
+use databend_common_meta_app::schema as mt;
+use databend_common_meta_app::share;
+use maplit::btreemap;
+use maplit::btreeset;
+use minitrace::func_name;
+
+use crate::common;
+
+#[test]
+fn test_decode_v74_table_meta() -> anyhow::Result<()> {
+    let table_meta_v74 = vec![
+        10, 223, 1, 10, 51, 10, 8, 110, 117, 108, 108, 97, 98, 108, 101, 18, 5, 97, 32, 43, 32, 51,
+        26, 26, 178, 2, 17, 154, 2, 8, 42, 0, 160, 6, 74, 168, 6, 24, 160, 6, 74, 168, 6, 24, 160,
+        6, 74, 168, 6, 24, 160, 6, 74, 168, 6, 24, 10, 27, 10, 6, 115, 116, 114, 105, 110, 103, 26,
+        9, 146, 2, 0, 160, 6, 74, 168, 6, 24, 32, 1, 160, 6, 74, 168, 6, 24, 10, 62, 10, 14, 118,
+        105, 114, 116, 117, 97, 108, 95, 115, 116, 114, 105, 110, 103, 26, 9, 146, 2, 0, 160, 6,
+        74, 168, 6, 24, 32, 2, 42, 25, 10, 17, 116, 111, 95, 98, 97, 115, 101, 54, 52, 40, 115,
+        116, 114, 105, 110, 103, 41, 160, 6, 74, 168, 6, 24, 160, 6, 74, 168, 6, 24, 10, 59, 10,
+        13, 115, 116, 111, 114, 101, 100, 95, 115, 116, 114, 105, 110, 103, 26, 9, 146, 2, 0, 160,
+        6, 74, 168, 6, 24, 32, 3, 42, 23, 18, 15, 114, 101, 118, 101, 114, 115, 101, 40, 115, 116,
+        114, 105, 110, 103, 41, 160, 6, 74, 168, 6, 24, 160, 6, 74, 168, 6, 24, 18, 6, 10, 1, 97,
+        18, 1, 98, 24, 4, 160, 6, 74, 168, 6, 24, 34, 10, 40, 97, 32, 43, 32, 50, 44, 32, 98, 41,
+        42, 10, 10, 3, 120, 121, 122, 18, 3, 102, 111, 111, 50, 2, 52, 52, 58, 10, 10, 3, 97, 98,
+        99, 18, 3, 100, 101, 102, 64, 0, 74, 10, 40, 97, 32, 43, 32, 50, 44, 32, 98, 41, 82, 7,
+        100, 101, 102, 97, 117, 108, 116, 162, 1, 23, 50, 48, 49, 52, 45, 49, 49, 45, 50, 56, 32,
+        49, 50, 58, 48, 48, 58, 48, 57, 32, 85, 84, 67, 170, 1, 23, 50, 48, 49, 52, 45, 49, 49, 45,
+        50, 57, 32, 49, 50, 58, 48, 48, 58, 49, 48, 32, 85, 84, 67, 178, 1, 13, 116, 97, 98, 108,
+        101, 95, 99, 111, 109, 109, 101, 110, 116, 186, 1, 6, 160, 6, 74, 168, 6, 24, 202, 1, 1,
+        99, 202, 1, 1, 99, 202, 1, 1, 99, 202, 1, 1, 99, 202, 1, 1, 99, 202, 1, 1, 99, 202, 1, 1,
+        99, 202, 1, 1, 99, 202, 1, 1, 99, 202, 1, 1, 99, 202, 1, 1, 99, 202, 1, 1, 99, 202, 1, 1,
+        99, 202, 1, 1, 99, 202, 1, 1, 99, 202, 1, 1, 99, 202, 1, 1, 99, 202, 1, 1, 99, 202, 1, 1,
+        99, 202, 1, 1, 99, 202, 1, 1, 99, 226, 1, 1, 1, 234, 1, 6, 10, 1, 97, 18, 1, 98, 160, 6,
+        74, 168, 6, 24,
+    ];
+
+    let want = || mt::TableMeta {
+        schema: Arc::new(ce::TableSchema::new_from(
+            vec![
+                ce::TableField::new(
+                    "nullable",
+                    ce::TableDataType::Nullable(Box::new(ce::TableDataType::Number(
+                        NumberDataType::Int8,
+                    ))),
+                )
+                .with_default_expr(Some("a + 3".to_string())),
+                ce::TableField::new("string", ce::TableDataType::String),
+                ce::TableField::new("virtual_string", ce::TableDataType::String)
+                    .with_computed_expr(Some(ComputedExpr::Virtual(
+                        "to_base64(string)".to_string(),
+                    ))),
+                ce::TableField::new("stored_string", ce::TableDataType::String)
+                    .with_computed_expr(Some(ComputedExpr::Stored("reverse(string)".to_string()))),
+            ],
+            btreemap! {s("a") => s("b")},
+        )),
+        catalog: "default".to_string(),
+        engine: "44".to_string(),
+        storage_params: None,
+        part_prefix: "".to_string(),
+        engine_options: btreemap! {s("abc") => s("def")},
+        options: btreemap! {s("xyz") => s("foo")},
+        default_cluster_key: Some("(a + 2, b)".to_string()),
+        cluster_keys: vec!["(a + 2, b)".to_string()],
+        default_cluster_key_id: Some(0),
+        created_on: Utc.with_ymd_and_hms(2014, 11, 28, 12, 0, 9).unwrap(),
+        updated_on: Utc.with_ymd_and_hms(2014, 11, 29, 12, 0, 10).unwrap(),
+        comment: s("table_comment"),
+        field_comments: vec!["c".to_string(); 21],
+        drop_on: None,
+        statistics: Default::default(),
+        shared_by: btreeset! {1},
+        column_mask_policy: Some(btreemap! {s("a") => s("b")}),
+    };
+    common::test_pb_from_to(func_name!(), want())?;
+    common::test_load_old(func_name!(), table_meta_v74.as_slice(), 74, want())?;
+
+    Ok(())
+}
+
+#[test]
+fn test_decode_v74_database_meta() -> anyhow::Result<()> {
+    let database_meta_v74 = vec![
+        34, 10, 10, 3, 120, 121, 122, 18, 3, 102, 111, 111, 42, 2, 52, 52, 50, 10, 10, 3, 97, 98,
+        99, 18, 3, 100, 101, 102, 162, 1, 23, 50, 48, 49, 52, 45, 49, 49, 45, 50, 56, 32, 49, 50,
+        58, 48, 48, 58, 48, 57, 32, 85, 84, 67, 170, 1, 23, 50, 48, 49, 52, 45, 49, 49, 45, 50, 57,
+        32, 49, 50, 58, 48, 48, 58, 48, 57, 32, 85, 84, 67, 178, 1, 7, 102, 111, 111, 32, 98, 97,
+        114, 202, 1, 21, 10, 6, 116, 101, 110, 97, 110, 116, 18, 5, 115, 104, 97, 114, 101, 160, 6,
+        74, 168, 6, 24, 160, 6, 74, 168, 6, 24,
+    ];
+
+    let want = || mt::DatabaseMeta {
+        engine: "44".to_string(),
+        engine_options: btreemap! {s("abc") => s("def")},
+        options: btreemap! {s("xyz") => s("foo")},
+        created_on: Utc.with_ymd_and_hms(2014, 11, 28, 12, 0, 9).unwrap(),
+        updated_on: Utc.with_ymd_and_hms(2014, 11, 29, 12, 0, 9).unwrap(),
+        comment: "foo bar".to_string(),
+        drop_on: None,
+        shared_by: BTreeSet::new(),
+        from_share: Some(share::ShareNameIdent {
+            tenant: "tenant".to_string(),
+            share_name: "share".to_string(),
+        }),
+    };
+
+    common::test_pb_from_to(func_name!(), want())?;
+    common::test_load_old(func_name!(), database_meta_v74.as_slice(), 74, want())
+}
+
+fn s(ss: impl ToString) -> String {
+    ss.to_string()
+}