diff --git a/src/rgw/driver/sfs/bucket.cc b/src/rgw/driver/sfs/bucket.cc index abd4cab67b99e..c5da77e2039cb 100644 --- a/src/rgw/driver/sfs/bucket.cc +++ b/src/rgw/driver/sfs/bucket.cc @@ -47,6 +47,11 @@ namespace rgw::sal { SFSBucket::SFSBucket(SFStore* _store, sfs::BucketRef _bucket) : StoreBucket(_bucket->get_info()), store(_store), bucket(_bucket) { + update_views(); +} + +void SFSBucket::update_views() { + get_info() = bucket->get_info(); set_attrs(bucket->get_attrs()); auto it = attrs.find(RGW_ATTR_ACL); @@ -56,6 +61,47 @@ SFSBucket::SFSBucket(SFStore* _store, sfs::BucketRef _bucket) } } +int SFSBucket::try_metadata_update( + const std::function& + apply_delta +) { + auto current_state = sfs::sqlite::DBOPBucketInfo(get_info(), get_attrs()); + auto db_conn = get_store().db_conn; + int res = + db_conn->transact([&](rgw::sal::sfs::sqlite::StorageRef storage) -> int { + auto db_state = sfs::get_meta_buckets(db_conn)->get_bucket( + bucket->get_bucket_id(), storage + ); + if (!db_state) { + // this is an error, the operation should not be retried + return -ERR_NO_SUCH_BUCKET; + } + if (current_state != *db_state) { + // the operation will be retried + return -ECANCELED; + } + // current_state == db_state, we apply the delta and we store the bucket. + int res = apply_delta(current_state); + if (res) { + return res; + } + sfs::get_meta_buckets(db_conn)->store_bucket(current_state, storage); + return 0; + }); + + if (!res) { + store->_refresh_buckets_safe(); + auto bref = store->get_bucket_ref(get_name()); + if (!bref) { + // if we go here, the state of this bucket is inconsistent + return -ERR_NO_SUCH_ENTITY; + } + bucket = bref; + update_views(); + } + return res; +} + void SFSBucket::write_meta(const DoutPrefixProvider* /*dpp*/) { // TODO } @@ -404,28 +450,12 @@ int SFSBucket:: int SFSBucket::merge_and_store_attrs( const DoutPrefixProvider* /*dpp*/, Attrs& new_attrs, optional_yield /*y*/ ) { - for (auto& it : new_attrs) { - attrs[it.first] = it.second; - - if (it.first == RGW_ATTR_ACL) { - auto lval = it.second.cbegin(); - acls.decode(lval); - } - } - for (auto& it : attrs) { - auto it_find = new_attrs.find(it.first); - if (it_find == new_attrs.end()) { - // this is an old attr that is not defined in the new_attrs - // delete it - attrs.erase(it.first); - } - } - - sfs::get_meta_buckets(get_store().db_conn) - ->store_bucket(sfs::sqlite::DBOPBucketInfo(get_info(), get_attrs())); - - store->_refresh_buckets_safe(); - return 0; + return try_metadata_update( + [&](sfs::sqlite::DBOPBucketInfo& current_state) -> int { + current_state.battrs = new_attrs; + return 0; + } + ); } // try_resolve_mp_from_oid tries to parse an integer id from oid to @@ -529,11 +559,22 @@ int SFSBucket::abort_multiparts( return sfs::SFSMultipartUploadV2::abort_multiparts(dpp, store, this); } +/** + * @brief Refresh this bucket object with the state obtained from the store. + Indeed it can happen that the state of this bucket is obsolete due to + concurrent threads updating metadata using their own SFSBucket instance. + */ int SFSBucket::try_refresh_info( const DoutPrefixProvider* dpp, ceph::real_time* /*pmtime*/ ) { - lsfs_warn(dpp) << __func__ << ": TODO" << dendl; - return -ENOTSUP; + auto bref = store->get_bucket_ref(get_name()); + if (!bref) { + lsfs_dout(dpp, 0) << fmt::format("no such bucket! {}", get_name()) << dendl; + return -ERR_NO_SUCH_BUCKET; + } + bucket = bref; + update_views(); + return 0; } int SFSBucket::read_usage( diff --git a/src/rgw/driver/sfs/bucket.h b/src/rgw/driver/sfs/bucket.h index dca868e5a1c85..fb8704ee8c172 100644 --- a/src/rgw/driver/sfs/bucket.h +++ b/src/rgw/driver/sfs/bucket.h @@ -62,6 +62,25 @@ class SFSBucket : public StoreBucket { SFSBucket(SFStore* _store, sfs::BucketRef _bucket); SFSBucket& operator=(const SFSBucket&) = delete; + /** + * This method updates the in-memory views of this object fetching + * from this.bucket. + * This method should be called every time this.bucket is updated + * from the backing storage. + * + * Views updated: + * + * - get_info() + * - get_attrs() + * - acls + */ + void update_views(); + + int try_metadata_update( + const std::function& + apply_delta + ); + virtual std::unique_ptr clone() override { return std::unique_ptr(new SFSBucket{*this}); } diff --git a/src/rgw/driver/sfs/sqlite/buckets/bucket_definitions.h b/src/rgw/driver/sfs/sqlite/buckets/bucket_definitions.h index 6d1b2e3e1c873..b0c8e7b2b2365 100644 --- a/src/rgw/driver/sfs/sqlite/buckets/bucket_definitions.h +++ b/src/rgw/driver/sfs/sqlite/buckets/bucket_definitions.h @@ -72,6 +72,16 @@ struct DBOPBucketInfo { DBOPBucketInfo(const DBOPBucketInfo& other) = default; DBOPBucketInfo& operator=(const DBOPBucketInfo& other) = default; + + bool operator==(const DBOPBucketInfo& other) const { + if (this->deleted != other.deleted) return false; + if (this->battrs != other.battrs) return false; + ceph::bufferlist this_binfo_bl; + this->binfo.encode(this_binfo_bl); + ceph::bufferlist other_binfo_bl; + other.binfo.encode(other_binfo_bl); + return this_binfo_bl == other_binfo_bl; + } }; using DBDeletedObjectItem = diff --git a/src/rgw/driver/sfs/sqlite/dbconn.h b/src/rgw/driver/sfs/sqlite/dbconn.h index 803878df04682..3cdf3bb2f7a77 100644 --- a/src/rgw/driver/sfs/sqlite/dbconn.h +++ b/src/rgw/driver/sfs/sqlite/dbconn.h @@ -264,6 +264,7 @@ class DBConn { std::vector sqlite_conns; const std::thread::id main_thread; mutable std::shared_mutex storage_pool_mutex; + mutable std::mutex transactional_block_mutex; public: CephContext* const cct; @@ -289,6 +290,12 @@ class DBConn { return dbapi::sqlite::database(get_storage()->filename()); } + int transact(const std::function& block) { + auto storage = get_storage(); + std::unique_lock lock(transactional_block_mutex); + return block(storage); + } + static std::string getDBPath(CephContext* cct) { auto rgw_sfs_path = cct->_conf.get_val("rgw_sfs_data_path"); auto db_path = diff --git a/src/rgw/driver/sfs/sqlite/sqlite_buckets.cc b/src/rgw/driver/sfs/sqlite/sqlite_buckets.cc index 28cc86737916f..8c230c95635a0 100644 --- a/src/rgw/driver/sfs/sqlite/sqlite_buckets.cc +++ b/src/rgw/driver/sfs/sqlite/sqlite_buckets.cc @@ -38,9 +38,11 @@ std::vector get_rgw_buckets( } std::optional SQLiteBuckets::get_bucket( - const std::string& bucket_id + const std::string& bucket_id, rgw::sal::sfs::sqlite::StorageRef storage ) const { - auto storage = conn->get_storage(); + if (!storage) { + storage = conn->get_storage(); + } auto bucket = storage->get_pointer(bucket_id); std::optional ret_value; if (bucket) { @@ -50,9 +52,11 @@ std::optional SQLiteBuckets::get_bucket( } std::optional> SQLiteBuckets::get_owner( - const std::string& bucket_id + const std::string& bucket_id, rgw::sal::sfs::sqlite::StorageRef storage ) const { - auto storage = conn->get_storage(); + if (!storage) { + storage = conn->get_storage(); + } const auto rows = storage->select( columns(&DBUser::user_id, &DBUser::display_name), inner_join(on(is_equal(&DBBucket::owner_id, &DBUser::user_id))), @@ -66,62 +70,92 @@ std::optional> SQLiteBuckets::get_owner( } std::vector SQLiteBuckets::get_bucket_by_name( - const std::string& bucket_name + const std::string& bucket_name, rgw::sal::sfs::sqlite::StorageRef storage ) const { - auto storage = conn->get_storage(); + if (!storage) { + storage = conn->get_storage(); + } return get_rgw_buckets( storage->get_all(where(c(&DBBucket::bucket_name) = bucket_name)) ); } -void SQLiteBuckets::store_bucket(const DBOPBucketInfo& bucket) const { - auto storage = conn->get_storage(); +void SQLiteBuckets::store_bucket( + const DBOPBucketInfo& bucket, rgw::sal::sfs::sqlite::StorageRef storage +) const { + if (!storage) { + storage = conn->get_storage(); + } auto db_bucket = get_db_bucket(bucket); storage->replace(db_bucket); } -void SQLiteBuckets::remove_bucket(const std::string& bucket_name) const { - auto storage = conn->get_storage(); +void SQLiteBuckets::remove_bucket( + const std::string& bucket_name, rgw::sal::sfs::sqlite::StorageRef storage +) const { + if (!storage) { + storage = conn->get_storage(); + } storage->remove(bucket_name); } -std::vector SQLiteBuckets::get_bucket_ids() const { - auto storage = conn->get_storage(); +std::vector SQLiteBuckets::get_bucket_ids( + rgw::sal::sfs::sqlite::StorageRef storage +) const { + if (!storage) { + storage = conn->get_storage(); + } return storage->select(&DBBucket::bucket_name); } std::vector SQLiteBuckets::get_bucket_ids( - const std::string& user_id + const std::string& user_id, rgw::sal::sfs::sqlite::StorageRef storage ) const { - auto storage = conn->get_storage(); + if (!storage) { + storage = conn->get_storage(); + } return storage->select( &DBBucket::bucket_name, where(c(&DBBucket::owner_id) = user_id) ); } -std::vector SQLiteBuckets::get_buckets() const { - auto storage = conn->get_storage(); +std::vector SQLiteBuckets::get_buckets( + rgw::sal::sfs::sqlite::StorageRef storage +) const { + if (!storage) { + storage = conn->get_storage(); + } return get_rgw_buckets(storage->get_all()); } std::vector SQLiteBuckets::get_buckets( - const std::string& user_id + const std::string& user_id, rgw::sal::sfs::sqlite::StorageRef storage ) const { - auto storage = conn->get_storage(); + if (!storage) { + storage = conn->get_storage(); + } return get_rgw_buckets( storage->get_all(where(c(&DBBucket::owner_id) = user_id)) ); } -std::vector SQLiteBuckets::get_deleted_buckets_ids() const { - auto storage = conn->get_storage(); +std::vector SQLiteBuckets::get_deleted_buckets_ids( + rgw::sal::sfs::sqlite::StorageRef storage +) const { + if (!storage) { + storage = conn->get_storage(); + } return storage->select( &DBBucket::bucket_id, where(c(&DBBucket::deleted) = true) ); } -bool SQLiteBuckets::bucket_empty(const std::string& bucket_id) const { - auto storage = conn->get_storage(); +bool SQLiteBuckets::bucket_empty( + const std::string& bucket_id, rgw::sal::sfs::sqlite::StorageRef storage +) const { + if (!storage) { + storage = conn->get_storage(); + } auto num_ids = storage->count( inner_join( on(is_equal(&DBObject::uuid, &DBVersionedObject::object_id)) @@ -136,9 +170,12 @@ bool SQLiteBuckets::bucket_empty(const std::string& bucket_id) const { } std::optional SQLiteBuckets::delete_bucket_transact( - const std::string& bucket_id, uint max_objects, bool& bucket_deleted + const std::string& bucket_id, uint max_objects, bool& bucket_deleted, + rgw::sal::sfs::sqlite::StorageRef storage ) const { - auto storage = conn->get_storage(); + if (!storage) { + storage = conn->get_storage(); + } RetrySQLiteBusy retry([&]() { bucket_deleted = false; DBDeletedObjectItems ret_values; @@ -186,9 +223,11 @@ std::optional SQLiteBuckets::delete_bucket_transact( } const std::optional SQLiteBuckets::get_stats( - const std::string& bucket_id + const std::string& bucket_id, rgw::sal::sfs::sqlite::StorageRef storage ) const { - auto storage = conn->get_storage(); + if (!storage) { + storage = conn->get_storage(); + } std::optional stats; auto res = storage->select( diff --git a/src/rgw/driver/sfs/sqlite/sqlite_buckets.h b/src/rgw/driver/sfs/sqlite/sqlite_buckets.h index 20cac98dee90c..3ef7a7baf516e 100644 --- a/src/rgw/driver/sfs/sqlite/sqlite_buckets.h +++ b/src/rgw/driver/sfs/sqlite/sqlite_buckets.h @@ -33,32 +33,62 @@ class SQLiteBuckets { uint64_t obj_count; }; - std::optional get_bucket(const std::string& bucket_id) const; - std::vector get_bucket_by_name(const std::string& bucket_name + std::optional get_bucket( + const std::string& bucket_id, + rgw::sal::sfs::sqlite::StorageRef storage = nullptr + ) const; + + std::vector get_bucket_by_name( + const std::string& bucket_name, + rgw::sal::sfs::sqlite::StorageRef storage = nullptr ) const; /// get_onwer returns bucket ownership information as a pair of /// (user id, display name) or nullopt std::optional> get_owner( - const std::string& bucket_id + const std::string& bucket_id, rgw::sal::sfs::sqlite::StorageRef storage = nullptr ) const; - void store_bucket(const DBOPBucketInfo& bucket) const; - void remove_bucket(const std::string& bucket_id) const; + void store_bucket( + const DBOPBucketInfo& bucket, + rgw::sal::sfs::sqlite::StorageRef storage = nullptr + ) const; - std::vector get_bucket_ids() const; - std::vector get_bucket_ids(const std::string& user_id) const; + void remove_bucket( + const std::string& bucket_id, + rgw::sal::sfs::sqlite::StorageRef storage = nullptr + ) const; - std::vector get_buckets() const; - std::vector get_buckets(const std::string& user_id) const; + std::vector get_bucket_ids( + rgw::sal::sfs::sqlite::StorageRef storage = nullptr + ) const; + std::vector get_bucket_ids( + const std::string& user_id, + rgw::sal::sfs::sqlite::StorageRef storage = nullptr + ) const; - std::vector get_deleted_buckets_ids() const; + std::vector get_buckets( + rgw::sal::sfs::sqlite::StorageRef storage = nullptr + ) const; + std::vector get_buckets( + const std::string& user_id, + rgw::sal::sfs::sqlite::StorageRef storage = nullptr + ) const; - bool bucket_empty(const std::string& bucket_id) const; + std::vector get_deleted_buckets_ids( + rgw::sal::sfs::sqlite::StorageRef storage = nullptr + ) const; + + bool bucket_empty( + const std::string& bucket_id, + rgw::sal::sfs::sqlite::StorageRef storage = nullptr + ) const; std::optional delete_bucket_transact( - const std::string& bucket_id, uint max_objects, bool& bucket_deleted + const std::string& bucket_id, uint max_objects, bool& bucket_deleted, + rgw::sal::sfs::sqlite::StorageRef storage = nullptr ) const; const std::optional get_stats( - const std::string& bucket_id + const std::string& bucket_id, + rgw::sal::sfs::sqlite::StorageRef storage = nullptr ) const; }; diff --git a/src/test/rgw/sfs/test_rgw_sfs_sfs_bucket.cc b/src/test/rgw/sfs/test_rgw_sfs_sfs_bucket.cc index e37781f2c0ab5..d57321f571ee7 100644 --- a/src/test/rgw/sfs/test_rgw_sfs_sfs_bucket.cc +++ b/src/test/rgw/sfs/test_rgw_sfs_sfs_bucket.cc @@ -15,6 +15,49 @@ #include "rgw/driver/sfs/sqlite/sqlite_users.h" #include "rgw/rgw_sal_sfs.h" +/* + These structs are in-memory mockable versions of actual structs/classes + that have a private rep. + Real types normally populate their rep via encode/decode methods. + For the sake of convenience, we define binary equivalent types with + public editable members. +*/ +namespace mockable { +struct DefaultRetention { + std::string mode; + int days; + int years; + + bool operator==(const DefaultRetention& other) const { + return this->mode == other.mode && this->days == other.days && + this->years == other.years; + } +}; + +struct ObjectLockRule { + mockable::DefaultRetention defaultRetention; + + bool operator==(const ObjectLockRule& other) const { + return this->defaultRetention == other.defaultRetention; + } +}; + +struct RGWObjectLock { + bool enabled; + bool rule_exist; + mockable::ObjectLockRule rule; + + bool operator==(const RGWObjectLock& other) const { + return this->enabled == other.enabled && + this->rule_exist == other.rule_exist && this->rule == other.rule; + } +}; + +mockable::RGWObjectLock& actual2mock(::RGWObjectLock& actual) { + return (mockable::RGWObjectLock&)actual; +} +} // namespace mockable + /* HINT s3gw.db will create here: /tmp/rgw_sfs_tests @@ -1438,6 +1481,7 @@ TEST_F(TestSFSBucket, ListNamespaceMultipartsBasics) { .path_uuid = uuid, .meta_str = "metastr", .mtime = now}; + int id = multipart.insert(mpop); ASSERT_GE(id, 0); @@ -1453,3 +1497,193 @@ TEST_F(TestSFSBucket, ListNamespaceMultipartsBasics) { EXPECT_EQ(results.objs[0].key.name, std::to_string(id)); EXPECT_EQ(results.objs[0].meta.mtime, now); } + +TEST_F(TestSFSBucket, RacedBucketMetadataWriteOperations) { + auto ceph_context = std::make_shared(CEPH_ENTITY_TYPE_CLIENT); + ceph_context->_conf.set_val("rgw_sfs_data_path", getTestDir()); + ceph_context->_log->start(); + auto store = new rgw::sal::SFStore(ceph_context.get(), getTestDir()); + + NoDoutPrefix ndp(ceph_context.get(), 1); + RGWEnv env; + env.init(ceph_context.get()); + createUser("usr_id", store->db_conn); + + rgw_user arg_user("", "usr_id", ""); + auto user = store->get_user(arg_user); + + rgw_bucket arg_bucket("t_id", "b_name", ""); + rgw_placement_rule arg_pl_rule("default", "STANDARD"); + std::string arg_swift_ver_location; + RGWQuotaInfo arg_quota_info; + RGWAccessControlPolicy arg_aclp_d = get_aclp_default(); + rgw::sal::Attrs arg_attrs; + + RGWBucketInfo arg_info = get_binfo(); + obj_version arg_objv; + bool existed = false; + req_info arg_req_info(ceph_context.get(), &env); + + std::unique_ptr bucket_from_create; + + EXPECT_EQ( + user->create_bucket( + &ndp, //dpp + arg_bucket, //b + "zg1", //zonegroup_id + arg_pl_rule, //placement_rule + arg_swift_ver_location, //swift_ver_location + &arg_quota_info, //pquota_info + arg_aclp_d, //policy + arg_attrs, //attrs + arg_info, //info + arg_objv, //ep_objv + false, //exclusive + false, //obj_lock_enabled + &existed, //existed + arg_req_info, //req_info + &bucket_from_create, //bucket + null_yield //optional_yield + ), + 0 + ); + + std::unique_ptr bucket_from_store_1; + + EXPECT_EQ( + store->get_bucket( + &ndp, user.get(), arg_info.bucket, &bucket_from_store_1, null_yield + ), + 0 + ); + + std::unique_ptr bucket_from_store_2; + + EXPECT_EQ( + store->get_bucket( + &ndp, user.get(), arg_info.bucket, &bucket_from_store_2, null_yield + ), + 0 + ); + + EXPECT_EQ(*bucket_from_store_1, *bucket_from_store_2); + + // merge_and_store_attrs + + rgw::sal::Attrs new_attrs; + RGWAccessControlPolicy arg_aclp = get_aclp_1(); + { + bufferlist acl_bl; + arg_aclp.encode(acl_bl); + new_attrs[RGW_ATTR_ACL] = acl_bl; + } + + EXPECT_EQ( + bucket_from_store_1->merge_and_store_attrs(&ndp, new_attrs, null_yield), 0 + ); + + // assert bucket_from_store_1 contains the RGW_ATTR_ACL attribute + auto acl_bl_1 = bucket_from_store_1->get_attrs().find(RGW_ATTR_ACL); + EXPECT_NE(bucket_from_store_1->get_attrs().end(), acl_bl_1); + + // assert bucket_from_store_2 does not contain the RGW_ATTR_ACL attribute + auto acl_bl_2 = bucket_from_store_2->get_attrs().find(RGW_ATTR_ACL); + EXPECT_EQ(bucket_from_store_2->get_attrs().end(), acl_bl_2); + + // put_info + + RGWObjectLock obj_lock; + mockable::RGWObjectLock& ol = mockable::actual2mock(obj_lock); + ol.enabled = true; + ol.rule.defaultRetention.years = 12; + ol.rule.defaultRetention.days = 31; + ol.rule.defaultRetention.mode = "GOVERNANCE"; + ol.rule_exist = true; + + bucket_from_store_2->get_info().obj_lock = obj_lock; + EXPECT_EQ(bucket_from_store_2->put_info(&ndp, false, real_time()), 0); + + auto& ol1 = mockable::actual2mock(bucket_from_store_1->get_info().obj_lock); + auto& ol2 = mockable::actual2mock(bucket_from_store_2->get_info().obj_lock); + + // obj lock structure in the respective memory cannot be equal at this point for the + // two bucket_from_store_1 and bucket_from_store_2 references; this simulates two threads updating + // the metadata over the same bucket using their own bucket reference (as it happens actually when 2 + // concurrent calls are issued from one or more S3 clients). + EXPECT_NE(ol1, ol2); + + // Getting now a third reference from the backing store should fetch an image equal to + // bucket_from_store_2 since that reference is the latest one that did a put_info(). + // merge_and_store_attrs() done with bucket_from_store_1 should now be lost due to + // bucket_from_store_2.put_info(). + std::unique_ptr bucket_from_store_3; + EXPECT_EQ( + store->get_bucket( + &ndp, user.get(), arg_info.bucket, &bucket_from_store_3, null_yield + ), + 0 + ); + + // ol2 and ol3 should be the same. + auto& ol3 = mockable::actual2mock(bucket_from_store_3->get_info().obj_lock); + EXPECT_EQ(ol2, ol3); + + // We expect to have lost RGW_ATTR_ACL attribute in the backing store. + auto acl_bl_3 = bucket_from_store_3->get_attrs().find(RGW_ATTR_ACL); + EXPECT_EQ(bucket_from_store_3->get_attrs().end(), acl_bl_3); + + // Now we repeat the updates interposing the try_refresh_info() on bucket_from_store_2. + // try_refresh_info() refreshes bucket_from_store_2's memory with the state obtained + // from the store. + EXPECT_EQ( + bucket_from_store_1->merge_and_store_attrs(&ndp, new_attrs, null_yield), 0 + ); + EXPECT_EQ(bucket_from_store_2->try_refresh_info(&ndp, nullptr), 0); + EXPECT_EQ(bucket_from_store_2->put_info(&ndp, false, real_time()), 0); + + // let's refetch bucket_from_store_3 from store. + EXPECT_EQ( + store->get_bucket( + &ndp, user.get(), arg_info.bucket, &bucket_from_store_3, null_yield + ), + 0 + ); + + // Now all the views over bucket_from_store_2, bucket_from_store_2 and + // bucket_from_store_3 should be the same, given that the + // underlying sfs::BucketRef are (hopefully) the same. + + // get_info() view + ol1 = mockable::actual2mock(bucket_from_store_1->get_info().obj_lock); + ol2 = mockable::actual2mock(bucket_from_store_2->get_info().obj_lock); + ol3 = mockable::actual2mock(bucket_from_store_3->get_info().obj_lock); + EXPECT_EQ(ol1, ol2); + EXPECT_EQ(ol2, ol3); + + // get_attrs() view and acls views + acl_bl_1 = bucket_from_store_1->get_attrs().find(RGW_ATTR_ACL); + EXPECT_NE(bucket_from_store_1->get_attrs().end(), acl_bl_1); + acl_bl_2 = bucket_from_store_2->get_attrs().find(RGW_ATTR_ACL); + EXPECT_NE(bucket_from_store_2->get_attrs().end(), acl_bl_2); + acl_bl_3 = bucket_from_store_3->get_attrs().find(RGW_ATTR_ACL); + EXPECT_NE(bucket_from_store_3->get_attrs().end(), acl_bl_3); + + { + RGWAccessControlPolicy aclp; + auto ci_lval = acl_bl_1->second.cbegin(); + aclp.decode(ci_lval); + EXPECT_EQ(aclp, arg_aclp); + } + { + RGWAccessControlPolicy aclp; + auto ci_lval = acl_bl_2->second.cbegin(); + aclp.decode(ci_lval); + EXPECT_EQ(aclp, arg_aclp); + } + { + RGWAccessControlPolicy aclp; + auto ci_lval = acl_bl_3->second.cbegin(); + aclp.decode(ci_lval); + EXPECT_EQ(aclp, arg_aclp); + } +}