LMDB exception normalization with mock client

cpp/arcticdb/CMakeLists.txt

@@ -433,6 +433,11 @@ set(arcticdb_srcs
         storage/failure_simulation.cpp
         storage/library_manager.cpp
         storage/azure/azure_storage.cpp
+        storage/lmdb/lmdb_client_wrapper.hpp
+        storage/lmdb/lmdb_mock_client.hpp
+        storage/lmdb/lmdb_mock_client.cpp
+        storage/lmdb/lmdb_real_client.hpp
+        storage/lmdb/lmdb_real_client.cpp
         storage/lmdb/lmdb_storage.cpp
         storage/file/mapped_file_storage.cpp
         storage/mongo/mongo_client.cpp

cpp/arcticdb/python/python_module.cpp

@@ -207,7 +207,7 @@ void register_error_code_ecosystem(py::module& m, py::exception<arcticdb::Arctic
 
     static py::exception<InternalException> internal_exception(m, "InternalException", compat_exception.ptr());
     static py::exception<StorageException> storage_exception(m, "StorageException", compat_exception.ptr());
-    static py::exception<::lmdb::map_full_error> lmdb_map_full_error(m, "LmdbMapFullError", storage_exception.ptr());
+    static py::exception<LMDBMapFullException> lmdb_map_full_exception(m, "LmdbMapFullError", storage_exception.ptr());
     static py::exception<UserInputException> user_input_exception(m, "UserInputException", compat_exception.ptr());
 
     py::register_exception_translator([](std::exception_ptr p) {
@@ -219,13 +219,13 @@ void register_error_code_ecosystem(py::module& m, py::exception<arcticdb::Arctic
             user_input_exception(e.what());
         } catch (const arcticdb::InternalException& e){
             internal_exception(e.what());
-        } catch (const ::lmdb::map_full_error& e) {
+        } catch (const LMDBMapFullException& e) {
             std::string msg = fmt::format("E5003: LMDB map is full. Close and reopen your LMDB backed Arctic instance with a "
                                           "larger map size. For example to open `/tmp/a/b/` with a map size of 5GB, "
                                           "use `Arctic(\"lmdb:///tmp/a/b?map_size=5GB\")`. Also see the "
                                           "[LMDB documentation](http://www.lmdb.tech/doc/group__mdb.html#gaa2506ec8dab3d969b0e609cd82e619e5). "
-                                          "LMDB info: code=[{}] origin=[{}] message=[{}]", e.code(), e.origin(), e.what());
-            lmdb_map_full_error(msg.c_str());
+                                          "LMDB info: message=[{}]", e.what());
+            lmdb_map_full_exception(msg.c_str());
         } catch (const StorageException& e) {
             storage_exception(e.what());
         } catch (const py::stop_iteration &e){

cpp/arcticdb/storage/lmdb/lmdb_client_wrapper.hpp

@@ -0,0 +1,65 @@
+/* Copyright 2024 Man Group Operations Limited
+ *
+ * Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt.
+ *
+ * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0.
+ */
+
+#pragma once
+
+#include <arcticdb/codec/segment.hpp>
+#include <arcticdb/entity/atom_key.hpp>
+#include <arcticdb/entity/variant_key.hpp>
+#include <arcticdb/storage/storage_utils.hpp>
+
+// LMDB++ is using `std::is_pod` in `lmdb++.h`, which is deprecated as of C++20.
+// See: https://github.com/drycpp/lmdbxx/blob/0b43ca87d8cfabba392dfe884eb1edb83874de02/lmdb%2B%2B.h#L1068
+// See: https://en.cppreference.com/w/cpp/types/is_pod
+// This suppresses the warning.
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#ifdef ARCTICDB_USING_CONDA
+#include <lmdb++.h>
+#else
+#include <third_party/lmdbxx/lmdb++.h>
+#endif
+#pragma GCC diagnostic pop
+
+
+namespace arcticdb::storage::lmdb {
+
+class LmdbClientWrapper {
+public:
+    virtual bool exists(
+            const std::string& db_name,
+            std::string& path,
+            ::lmdb::txn& txn,
+            ::lmdb::dbi& dbi) const = 0;
+
+    virtual std::optional<Segment> read(
+            const std::string& db_name,
+            std::string& path,
+            ::lmdb::txn& txn,
+            ::lmdb::dbi& dbi) const = 0;
+
+    virtual void write(
+            const std::string& db_name,
+            std::string& path,
+            Segment&& segment,
+            ::lmdb::txn& txn,
+            ::lmdb::dbi& dbi,
+            int64_t overwrite_flag) = 0;
+
+    virtual bool remove(const std::string& db_name, std::string& path, ::lmdb::txn& txn, ::lmdb::dbi& dbi) = 0;
+
+    virtual std::vector<VariantKey> list(
+            const std::string& db_name,
+            const std::string& prefix,
+            ::lmdb::txn& txn,
+            ::lmdb::dbi& dbi,
+            KeyType key_type) const = 0;
+
+    virtual ~LmdbClientWrapper() = default;
+};
+
+} // namespace arcticdb::storage::lmdb

cpp/arcticdb/storage/lmdb/lmdb_mock_client.cpp

@@ -0,0 +1,135 @@
+/* Copyright 2024 Man Group Operations Limited
+ *
+ * Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt.
+ *
+ * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0.
+ */
+
+#include <arcticdb/storage/lmdb/lmdb_mock_client.hpp>
+
+#include <arcticdb/codec/segment.hpp>
+#include <arcticdb/entity/atom_key.hpp>
+#include <arcticdb/entity/serialized_key.hpp>
+#include <arcticdb/util/string_utils.hpp>
+
+
+namespace arcticdb::storage::lmdb {
+
+std::string MockLmdbClient::get_failure_trigger(
+        const std::string& path,
+        StorageOperation operation_to_fail,
+        int error_code) {
+    return fmt::format("{}#Failure_{}_{}", path, operation_to_string(operation_to_fail), error_code);
+}
+
+std::string lmdb_operation_string(StorageOperation operation) {
+    switch (operation) {
+        case StorageOperation::READ:
+            return "mdb_get";
+        case StorageOperation::WRITE:
+            return "mdb_put";
+        case StorageOperation::DELETE:
+            return "mdb_del";
+        case StorageOperation::LIST:
+            return "mdb_cursor_get";
+        case StorageOperation::EXISTS:
+            return "mdb_get";
+        default:
+            return "unknown";
+    }
+}
+
+void raise_if_has_failure_trigger(const LmdbKey& key, StorageOperation operation) {
+    auto path = key.path;
+    auto failure_string_for_operation = "#Failure_" + operation_to_string(operation) + "_";
+    auto position = path.rfind(failure_string_for_operation);
+
+    if (position == std::string::npos) {
+        return;
+    }
+
+    int error_code = 0;
+    try {
+        auto start = position + failure_string_for_operation.size();
+        error_code = stoi(path.substr(start));
+        auto error_message = fmt::format("Simulated Error, message: operation {}, error code {}",
+                                         operation_to_string(operation), error_code);
+    } catch (std::exception&) {
+        return;
+    }
+
+    if (error_code != 0) {
+        ::lmdb::error::raise(lmdb_operation_string(operation).data(), error_code);
+    }
+}
+
+void raise_key_exists_error(std::string lmdb_op) {
+    ::lmdb::error::raise(lmdb_op.data(), MDB_KEYEXIST);
+}
+
+bool MockLmdbClient::has_key(const LmdbKey& key) const {
+    return lmdb_contents.find(key) != lmdb_contents.end();
+}
+
+bool MockLmdbClient::exists(const std::string& db_name, std::string& path, ::lmdb::txn&, ::lmdb::dbi&) const {
+    LmdbKey key = {db_name, path};
+    raise_if_has_failure_trigger(key, StorageOperation::EXISTS);
+
+    return has_key(key);
+}
+
+std::optional<Segment> MockLmdbClient::read(const std::string& db_name, std::string& path, ::lmdb::txn&, ::lmdb::dbi&) const {
+    LmdbKey key = {db_name, path};
+    raise_if_has_failure_trigger(key, StorageOperation::READ);
+
+    if (!has_key(key)) {
+        return std::nullopt;
+    }
+
+    return lmdb_contents.at(key);
+}
+
+void MockLmdbClient::write(const std::string& db_name, std::string& path, arcticdb::Segment&& segment,
+                           ::lmdb::txn&, ::lmdb::dbi&, int64_t) {
+    LmdbKey key = {db_name, path};
+    raise_if_has_failure_trigger(key, StorageOperation::WRITE);
+
+    if(has_key(key)) {
+        raise_key_exists_error(lmdb_operation_string(StorageOperation::WRITE));
+    } else {
+        lmdb_contents.insert({key, segment});
+    }
+}
+
+bool MockLmdbClient::remove(const std::string& db_name, std::string& path, ::lmdb::txn&, ::lmdb::dbi&) {
+    LmdbKey key = {db_name, path};
+    raise_if_has_failure_trigger(key, StorageOperation::DELETE);
+
+    if (!has_key(key)) {
+        return false;
+    }
+
+    lmdb_contents.erase(key);
+    return true;
+}
+
+std::vector<VariantKey> MockLmdbClient::list(const std::string& db_name, const std::string& prefix, ::lmdb::txn&,
+                                             ::lmdb::dbi&, KeyType key_type) const {
+    std::vector<VariantKey> found_keys;
+
+    for (const auto& [key, segment] : lmdb_contents) {
+        if (key.db_name == db_name && util::string_starts_with(prefix, key.path)) {
+            raise_if_has_failure_trigger(key, StorageOperation::LIST);
+
+            auto k = variant_key_from_bytes(
+                    reinterpret_cast<const uint8_t *>(key.path.data()),
+                    key.path.size(),
+                    key_type);
+            found_keys.push_back(k);
+        }
+    }
+
+    return found_keys;
+}
+
+} // namespace arcticdb::storage::lmdb

cpp/arcticdb/storage/lmdb/lmdb_mock_client.hpp

@@ -0,0 +1,72 @@
+/* Copyright 2024 Man Group Operations Limited
+ *
+ * Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt.
+ *
+ * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0.
+ */
+
+#pragma once
+
+#include <arcticdb/storage/lmdb/lmdb_client_wrapper.hpp>
+
+#include <arcticdb/codec/segment.hpp>
+#include <arcticdb/entity/atom_key.hpp>
+#include <arcticdb/entity/variant_key.hpp>
+#include <arcticdb/storage/storage_mock_client.hpp>
+#include <arcticdb/storage/storage_utils.hpp>
+
+
+namespace arcticdb::storage::lmdb {
+
+struct LmdbKey {
+    std::string db_name;
+    std::string path;
+
+    bool operator<(const LmdbKey& other) const {
+        return std::tie(db_name, path) < std::tie(other.db_name, other.path);
+    }
+};
+
+class MockLmdbClient : public LmdbClientWrapper {
+public:
+    static std::string get_failure_trigger(
+            const std::string& path,
+            StorageOperation operation_to_fail,
+            int error_code);
+
+    bool exists(
+            const std::string& db_name,
+            std::string& path,
+            ::lmdb::txn& txn,
+            ::lmdb::dbi& dbi) const override;
+
+    std::optional<Segment> read(
+            const std::string& db_name,
+            std::string& path,
+            ::lmdb::txn& txn,
+            ::lmdb::dbi& dbi) const override;
+
+    void write(
+            const std::string& db_name,
+            std::string& path,
+            Segment&& segment,
+            ::lmdb::txn& txn,
+            ::lmdb::dbi& dbi,
+            int64_t overwrite_flag) override;
+
+    bool remove(const std::string& db_name, std::string& path, ::lmdb::txn& txn, ::lmdb::dbi& dbi) override;
+
+    std::vector<VariantKey> list(
+            const std::string& db_name,
+            const std::string& prefix,
+            ::lmdb::txn& txn,
+            ::lmdb::dbi& dbi,
+            KeyType key_type) const override;
+
+private:
+    std::map<LmdbKey, Segment> lmdb_contents;
+
+    bool has_key(const LmdbKey& key) const;
+};
+
+} // namespace arcticdb::storage::lmdb