From 768a39b2416b654096f531236241d2d60785355c Mon Sep 17 00:00:00 2001 From: violeta Date: Mon, 29 Aug 2022 13:31:45 +0400 Subject: [PATCH 01/30] Minor: code correction. --- src/backend_stl.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/backend_stl.cpp b/src/backend_stl.cpp index afa5c4e3e..b3c076d59 100644 --- a/src/backend_stl.cpp +++ b/src/backend_stl.cpp @@ -795,8 +795,10 @@ void ukv_col_drop( if (c_mode == ukv_col_drop_keys_vals_handle_k) db.named.erase(col_it); - else if (c_mode == ukv_col_drop_keys_vals_k) - col.pairs.clear(), col.unique_elements = 0; + else if (c_mode == ukv_col_drop_keys_vals_k) { + col.pairs.clear(); + col.unique_elements = 0; + } else if (c_mode == ukv_col_drop_vals_k) { generation_t gen = ++db.youngest_generation; From f4b4b0fc4c75245bc8654cf3f8918cf5f2c9b2fe Mon Sep 17 00:00:00 2001 From: violeta Date: Tue, 30 Aug 2022 11:33:52 +0400 Subject: [PATCH 02/30] Extending support for ukv_col_drop --- include/ukv/cpp/db.hpp | 12 ++++++++++++ include/ukv/cpp/graph_ref.hpp | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/include/ukv/cpp/db.hpp b/include/ukv/cpp/db.hpp index 3364d0c5f..82bcabc9b 100644 --- a/include/ukv/cpp/db.hpp +++ b/include/ukv/cpp/db.hpp @@ -121,6 +121,18 @@ class col_t { return (maybe->min + maybe->max) / 2; } + status_t clear() noexcept { + status_t status; + ukv_col_drop(db_, nullptr, col_, ukv_col_drop_keys_vals_k, status.member_ptr()); + return status; + } + + status_t clear_values() noexcept { + status_t status; + ukv_col_drop(db_, nullptr, col_, ukv_col_drop_vals_k, status.member_ptr()); + return status; + } + inline members_ref_gt operator[](std::initializer_list keys) noexcept { return at(keys); } inline members_ref_gt at(std::initializer_list keys) noexcept { // return at(strided_range(keys)); diff --git a/include/ukv/cpp/graph_ref.hpp b/include/ukv/cpp/graph_ref.hpp index 49c52f799..4a7351113 100644 --- a/include/ukv/cpp/graph_ref.hpp +++ b/include/ukv/cpp/graph_ref.hpp @@ -114,6 +114,18 @@ class graph_ref_t { return status; } + status_t graph_ref_t::clear_edges() noexcept { + status_t status; + ukv_col_drop(db_, nullptr, col_, ukv_col_drop_vals_k, status.member_ptr()); + return status; + } + + status_t graph_ref_t::clear() noexcept { + status_t status; + ukv_col_drop(db_, nullptr, col_, ukv_col_drop_keys_vals_k, status.member_ptr()); + return status; + } + expected_gt degree( // ukv_key_t vertex, ukv_vertex_role_t role = ukv_vertex_role_any_k, From 2c5445329ddd6b7a2191eb50221adff1fd972294 Mon Sep 17 00:00:00 2001 From: violeta Date: Tue, 30 Aug 2022 15:26:01 +0400 Subject: [PATCH 03/30] Approximate implementation ukv_col_drop for backend rocksdb and leveldb. --- src/backend_leveldb.cpp | 43 ++++++++++++++++++++++++++++++++ src/backend_rocksdb.cpp | 54 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+) diff --git a/src/backend_leveldb.cpp b/src/backend_leveldb.cpp index f8e8e1706..5c7f0e64b 100644 --- a/src/backend_leveldb.cpp +++ b/src/backend_leveldb.cpp @@ -524,6 +524,49 @@ void ukv_col_upsert( // *c_error = "Collections not supported by LevelDB!"; } +void ukv_col_drop( + // Inputs: + ukv_t const c_db, + ukv_str_view_t c_col_name, + ukv_col_t c_col_id, + ukv_col_drop_mode_t c_mode, + // Outputs: + ukv_error_t* c_error) { + + return_if_error(c_db, c_error, uninitialized_state_k, "DataBase is uninitialized"); + + auto col_name = c_col_name ? std::string_view(c_col_name) : std::string_view(); + bool invalidate = c_mode == ukv_col_drop_keys_vals_handle_k; + return_if_error(!col_name.empty() || !invalidate, + c_error, + args_combo_k, + "Default collection can't be invlaidated."); + + level_db_t& db = *reinterpret_cast(c_db); + + if (c_mode == ukv_col_drop_keys_vals_handle_k) { + // TODO + } + + else if (c_mode == ukv_col_drop_keys_vals_k) { + leveldb::WriteBatch batch; + auto it = std::unique_ptr(db.NewIterator(leveldb::ReadOptions())); + for (it->SeekToFirst(); it->Valid(); it->Next()) + batch.Delete(it->key()); + level_status_t status = db.Write(leveldb::WriteOptions(), &batch); + export_error(status, c_error); + } + + else if (c_mode == ukv_col_drop_vals_k) { + leveldb::WriteBatch batch; + auto it = std::unique_ptr(db.NewIterator(leveldb::ReadOptions())); + for (it->SeekToFirst(); it->Valid(); it->Next()) + batch.Put(it->key(), 0); + level_status_t status = db.Write(leveldb::WriteOptions(), &batch); + export_error(status, c_error); + } +} + void ukv_col_remove( // ukv_t const c_db, ukv_str_view_t c_col_name, diff --git a/src/backend_rocksdb.cpp b/src/backend_rocksdb.cpp index 9997eded5..77c8d5227 100644 --- a/src/backend_rocksdb.cpp +++ b/src/backend_rocksdb.cpp @@ -649,6 +649,60 @@ void ukv_col_upsert( } } +void ukv_col_drop( + // Inputs: + ukv_t const c_db, + ukv_str_view_t c_col_name, + ukv_col_t c_col_id, + ukv_col_drop_mode_t c_mode, + // Outputs: + ukv_error_t* c_error) { + + return_if_error(c_db, c_error, uninitialized_state_k, "DataBase is uninitialized"); + + auto col_name = c_col_name ? std::string_view(c_col_name) : std::string_view(); + bool invalidate = c_mode == ukv_col_drop_keys_vals_handle_k; + return_if_error(!col_name.empty() || !invalidate, + c_error, + args_combo_k, + "Default collection can't be invlaidated."); + + rocks_db_t& db = *reinterpret_cast(c_db); + if (c_mode == ukv_col_drop_keys_vals_handle_k) { + for (auto it = db.columns.begin(); it != db.columns.end(); it++) { + if (c_col_name == (*it)->GetName() && (*it)->GetName() != "default") { + rocks_status_t status = db.native->DropColumnFamily(*it); + if (export_error(status, c_error)) + return; + db.columns.erase(it--); + break; + } + } + } + + else if (c_mode == ukv_col_drop_keys_vals_k) { + rocksdb::WriteBatch batch; + auto col = db.native->DefaultColumnFamily(); + auto it = std::unique_ptr(db.native->NewIterator(rocksdb::ReadOptions(), col)); + for (it->SeekToFirst(); it->Valid(); it->Next()) + batch.Delete(col, it->key()); + rocks_status_t status = db.native->Write(rocksdb::WriteOptions(), &batch); + export_error(status, c_error); + return; + } + + else if (c_mode == ukv_col_drop_vals_k) { + rocksdb::WriteBatch batch; + auto col = db.native->DefaultColumnFamily(); + auto it = std::unique_ptr(db.native->NewIterator(rocksdb::ReadOptions(), col)); + for (it->SeekToFirst(); it->Valid(); it->Next()) + batch.Put(col, it->key(), 0); + rocks_status_t status = db.native->Write(rocksdb::WriteOptions(), &batch); + export_error(status, c_error); + return; + } +} + void ukv_col_remove( // ukv_t const c_db, ukv_str_view_t c_col_name, From 6c49d5c2ab88fbc443c5a77a016f09420a02975f Mon Sep 17 00:00:00 2001 From: violeta Date: Tue, 30 Aug 2022 16:17:01 +0400 Subject: [PATCH 04/30] Minor --- include/ukv/cpp/graph_ref.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/ukv/cpp/graph_ref.hpp b/include/ukv/cpp/graph_ref.hpp index 4a7351113..91deab465 100644 --- a/include/ukv/cpp/graph_ref.hpp +++ b/include/ukv/cpp/graph_ref.hpp @@ -114,13 +114,13 @@ class graph_ref_t { return status; } - status_t graph_ref_t::clear_edges() noexcept { + status_t clear_edges() noexcept { status_t status; ukv_col_drop(db_, nullptr, col_, ukv_col_drop_vals_k, status.member_ptr()); return status; } - status_t graph_ref_t::clear() noexcept { + status_t clear() noexcept { status_t status; ukv_col_drop(db_, nullptr, col_, ukv_col_drop_keys_vals_k, status.member_ptr()); return status; From 8fa4da8f5825af1c61d26d21304d1245016c4f28 Mon Sep 17 00:00:00 2001 From: violeta Date: Tue, 30 Aug 2022 16:53:12 +0400 Subject: [PATCH 05/30] Add test --- src/test.cpp | 101 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/src/test.cpp b/src/test.cpp index de84a38aa..85fd1c04a 100644 --- a/src/test.cpp +++ b/src/test.cpp @@ -18,6 +18,45 @@ using namespace unum::ukv; using namespace unum; +size_t vertexes_count = 1000; +size_t next_connect = 100; +std::vector es; + +edge_t make_edge(ukv_key_t edge_id, ukv_key_t v1, ukv_key_t v2) { + edge_t e; + e.id = edge_id; + e.source_id = v1; + e.target_id = v2; + return e; +} + +void fill_edges() { + es.clear(); + size_t edge_id = 0; + for (size_t vertex_id = 0; vertex_id < vertexes_count; ++vertex_id) { + size_t connect_with = vertex_id + next_connect; + while (connect_with < vertexes_count) { + edge_id++; + es.push_back(make_edge(edge_id, vertex_id, connect_with)); + connect_with = connect_with + next_connect; + } + } +} + +void upsert_edge(graph_ref_t& graph) { + size_t edge_id = 0; + for (size_t vertex_id = 0; vertex_id < vertexes_count; ++vertex_id) { + size_t connect_with = vertex_id + next_connect; + while (connect_with < vertexes_count) { + edge_id++; + graph.upsert(make_edge(edge_id, vertex_id, connect_with)); + EXPECT_TRUE(*graph.contains(vertex_id)); + EXPECT_EQ(*graph.degree(vertex_id), connect_with / 100u); + connect_with += next_connect; + } + } +} + #define macro_concat_(prefix, suffix) prefix##suffix #define macro_concat(prefix, suffix) macro_concat_(prefix, suffix) #define _ [[maybe_unused]] auto macro_concat(_, __LINE__) @@ -672,6 +711,68 @@ TEST(db, net_batch) { EXPECT_TRUE(db.clear()); } +TEST(db, upsert_edge) { + db_t db; + EXPECT_TRUE(db.open("")); + + col_t main = *db.collection(); + graph_ref_t graph = main.as_graph(); + upsert_edge(graph); +} + +TEST(db, upsert_edges) { + db_t db; + EXPECT_TRUE(db.open("")); + + col_t main = *db.collection(); + graph_ref_t graph = main.as_graph(); + fill_edges(); + EXPECT_TRUE(graph.upsert(edges(es))); + for (size_t vertex_id = 0; vertex_id < vertexes_count; ++vertex_id) { + EXPECT_TRUE(*graph.contains(vertex_id)); + EXPECT_EQ(*graph.degree(vertex_id), 9u); + } +} + +TEST(db, remove_vertexes) { + db_t db; + EXPECT_TRUE(db.open("")); + + col_t main = *db.collection(); + graph_ref_t graph = main.as_graph(); + fill_edges(); + EXPECT_TRUE(graph.upsert(edges(es))); + + for (size_t vertex_id = 0; vertex_id < vertexes_count; ++vertex_id) { + EXPECT_TRUE(*graph.contains(vertex_id)); + EXPECT_TRUE(graph.remove(vertex_id)); + EXPECT_FALSE(*graph.contains(vertex_id)); + } +} + +TEST(db, remove_edge) { + db_t db; + EXPECT_TRUE(db.open("")); + + col_t main = *db.collection(); + graph_ref_t graph = main.as_graph(); + fill_edges(); + EXPECT_TRUE(graph.upsert(edges(es))); + + size_t edge_id = 0; + for (size_t vertex_id = 0; vertex_id < vertexes_count; ++vertex_id) { + size_t connect_with = vertex_id + next_connect; + while (connect_with < vertexes_count) { + edge_id++; + graph.remove(make_edge(edge_id, vertex_id, connect_with)); + connect_with = connect_with + next_connect; + } + } + + for (size_t vertex_id = 0; vertex_id < vertexes_count; ++vertex_id) + EXPECT_TRUE(*graph.contains(vertex_id)); +} + int main(int argc, char** argv) { std::filesystem::create_directory("./tmp"); ::testing::InitGoogleTest(&argc, argv); From 182709a82af52c772d1e65fa5ec00becc67eb0cf Mon Sep 17 00:00:00 2001 From: violeta Date: Tue, 30 Aug 2022 18:00:46 +0400 Subject: [PATCH 06/30] Build fix --- src/backend_leveldb.cpp | 12 ++++++------ src/backend_rocksdb.cpp | 22 +++++++++++----------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/backend_leveldb.cpp b/src/backend_leveldb.cpp index 5c7f0e64b..d5cad0c32 100644 --- a/src/backend_leveldb.cpp +++ b/src/backend_leveldb.cpp @@ -114,8 +114,8 @@ void write_one( // ukv_error_t* c_error) { auto task = tasks[0]; - auto key = to_slice(task.key); - level_status_t status = task.is_deleted() ? db.Delete(options, key) : db.Put(options, key, to_slice(task.view())); + auto key = to_slice(task); + level_status_t status = task ? db.Delete(options, key) : db.Put(options, key, to_slice(task)); export_error(status, c_error); } @@ -128,11 +128,11 @@ void write_many( // leveldb::WriteBatch batch; for (std::size_t i = 0; i != tasks.size(); ++i) { auto task = tasks[i]; - auto key = to_slice(task.key); - if (task.is_deleted()) + auto key = to_slice(task); + if (task) batch.Delete(key); else - batch.Put(key, to_slice(task.view())); + batch.Put(key, to_slice(task)); } level_status_t status = db.Write(options, &batch); @@ -174,7 +174,7 @@ void ukv_write( // strided_iterator_gt vals {c_vals, c_vals_stride}; strided_iterator_gt offs {c_offs, c_offs_stride}; strided_iterator_gt lens {c_lens, c_lens_stride}; - contents_arg_t tasks {cols, keys, vals, offs, lens, c_tasks_count}; + contents_arg_t tasks {vals, offs, lens, cols, c_tasks_count}; leveldb::WriteOptions options; if (c_options & ukv_option_write_flush_k) diff --git a/src/backend_rocksdb.cpp b/src/backend_rocksdb.cpp index 77c8d5227..df3b81e07 100644 --- a/src/backend_rocksdb.cpp +++ b/src/backend_rocksdb.cpp @@ -148,17 +148,17 @@ void write_one( // auto task = tasks[0]; auto col = rocks_collection(db, task.col); - auto key = to_slice(task.key); + auto key = to_slice(task); rocks_status_t status; if (txn) - status = task.is_deleted() // + status = task // ? txn->SingleDelete(col, key) - : txn->Put(col, key, to_slice(task.view())); + : txn->Put(col, key, to_slice(task)); else - status = task.is_deleted() // + status = task // ? db.native->SingleDelete(options, col, key) - : db.native->Put(options, col, key, to_slice(task.view())); + : db.native->Put(options, col, key, to_slice(task)); export_error(status, c_error); } @@ -174,10 +174,10 @@ void write_many( // for (std::size_t i = 0; i != tasks.size(); ++i) { auto task = tasks[i]; auto col = rocks_collection(db, task.col); - auto key = to_slice(task.key); - auto status = task.is_deleted() // + auto key = to_slice(task); + auto status = task // ? txn->Delete(col, key) - : txn->Put(col, key, to_slice(task.view())); + : txn->Put(col, key, to_slice(task)); export_error(status, c_error); } } @@ -186,10 +186,10 @@ void write_many( // for (std::size_t i = 0; i != tasks.size(); ++i) { auto task = tasks[i]; auto col = rocks_collection(db, task.col); - auto key = to_slice(task.key); - auto status = task.is_deleted() // + auto key = to_slice(task); + auto status = task // ? batch.Delete(col, key) - : batch.Put(col, key, to_slice(task.view())); + : batch.Put(col, key, to_slice(task)); export_error(status, c_error); } From 79ac1cfc8027cfacb89580ec152eb939ad950b26 Mon Sep 17 00:00:00 2001 From: violeta Date: Tue, 30 Aug 2022 18:08:23 +0400 Subject: [PATCH 07/30] Build fix --- python/pybind/crud.hpp | 4 ++-- src/backend_leveldb.cpp | 8 ++++---- src/backend_rocksdb.cpp | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/python/pybind/crud.hpp b/python/pybind/crud.hpp index c9be021c7..e315be885 100644 --- a/python/pybind/crud.hpp +++ b/python/pybind/crud.hpp @@ -140,7 +140,7 @@ static py::object has_one_binary(py_col_t& col, PyObject* key_py) { ukv_val_ptr_t found_values = nullptr; ukv_val_len_t* found_offsets = nullptr; ukv_val_len_t* found_lengths = nullptr; - auto options = static_cast(col.options() | ukv_option_read_lengths_k); + auto options = static_cast(col.options() /* | ukv_option_read_lengths_k */); { [[maybe_unused]] py::gil_scoped_release release; @@ -209,7 +209,7 @@ static py::object has_many_binaries(py_col_t& col, PyObject* keys_py) { ukv_val_ptr_t found_values = nullptr; ukv_val_len_t* found_offsets = nullptr; ukv_val_len_t* found_lengths = nullptr; - auto options = static_cast(col.options() | ukv_option_read_lengths_k); + auto options = static_cast(col.options() /* | ukv_option_read_lengths_k */); std::vector keys; py_transform_n(keys_py, &py_to_scalar, std::back_inserter(keys)); diff --git a/src/backend_leveldb.cpp b/src/backend_leveldb.cpp index d5cad0c32..14ecdb8fe 100644 --- a/src/backend_leveldb.cpp +++ b/src/backend_leveldb.cpp @@ -174,7 +174,7 @@ void ukv_write( // strided_iterator_gt vals {c_vals, c_vals_stride}; strided_iterator_gt offs {c_offs, c_offs_stride}; strided_iterator_gt lens {c_lens, c_lens_stride}; - contents_arg_t tasks {vals, offs, lens, cols, c_tasks_count}; + contents_arg_t tasks {vals, offs, lens, , c_tasks_count}; leveldb::WriteOptions options; if (c_options & ukv_option_write_flush_k) @@ -360,11 +360,11 @@ void ukv_read( // try { if (c_tasks_count == 1) { - auto func = (c_options & ukv_option_read_lengths_k) ? &measure_one : &read_one; + auto func = (c_options /* & ukv_option_read_lengths_k */) ? &measure_one : &read_one; func(db, tasks, options, value, c_found_values, c_found_offsets, c_found_lengths, arena, c_error); } else { - auto func = (c_options & ukv_option_read_lengths_k) ? &measure_many : &read_many; + auto func = (c_options /* & ukv_option_read_lengths_k */) ? &measure_many : &read_many; func(db, tasks, options, value, c_found_values, c_found_offsets, c_found_lengths, arena, c_error); } } @@ -407,7 +407,7 @@ void ukv_scan( // strided_iterator_gt lens {c_scan_lengths, c_scan_lengths_stride}; scans_arg_t tasks {{}, keys, lens, c_min_tasks_count}; - bool export_lengths = (c_options & ukv_option_read_lengths_k); + bool export_lengths = (c_options /* & ukv_option_read_lengths_k */); leveldb::ReadOptions options; options.fill_cache = false; diff --git a/src/backend_rocksdb.cpp b/src/backend_rocksdb.cpp index df3b81e07..3e4ce9c4f 100644 --- a/src/backend_rocksdb.cpp +++ b/src/backend_rocksdb.cpp @@ -451,11 +451,11 @@ void ukv_read( // try { if (c_tasks_count == 1) { - auto func = (c_options & ukv_option_read_lengths_k) ? &measure_one : &read_one; + auto func = (c_options /* & ukv_option_read_lengths_k) */) ? &measure_one : &read_one; func(db, txn, tasks, options, c_found_values, c_found_offsets, c_found_lengths, arena, c_error); } else { - auto func = (c_options & ukv_option_read_lengths_k) ? &measure_many : &read_many; + auto func = (c_options /* & ukv_option_read_lengths_k */) ? &measure_many : &read_many; func(db, txn, tasks, options, c_found_values, c_found_offsets, c_found_lengths, arena, c_error); } } @@ -501,7 +501,7 @@ void ukv_scan( // strided_iterator_gt lengths {c_scan_lengths, c_scan_lengths_stride}; scans_arg_t tasks {cols, keys, lengths, c_min_tasks_count}; - bool export_lengths = (c_options & ukv_option_read_lengths_k); + bool export_lengths = (c_options /* & ukv_option_read_lengths_k */); rocksdb::ReadOptions options; options.fill_cache = false; From 0a858952e737294c6e07c7e501b859c8e8c2e4c4 Mon Sep 17 00:00:00 2001 From: violeta Date: Tue, 30 Aug 2022 18:15:19 +0400 Subject: [PATCH 08/30] Build fix --- src/backend_leveldb.cpp | 4 ++-- src/backend_rocksdb.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/backend_leveldb.cpp b/src/backend_leveldb.cpp index 14ecdb8fe..6d2f64e42 100644 --- a/src/backend_leveldb.cpp +++ b/src/backend_leveldb.cpp @@ -174,7 +174,7 @@ void ukv_write( // strided_iterator_gt vals {c_vals, c_vals_stride}; strided_iterator_gt offs {c_offs, c_offs_stride}; strided_iterator_gt lens {c_lens, c_lens_stride}; - contents_arg_t tasks {vals, offs, lens, , c_tasks_count}; + contents_arg_t tasks {vals, offs, lens, {}, c_tasks_count}; leveldb::WriteOptions options; if (c_options & ukv_option_write_flush_k) @@ -353,7 +353,7 @@ void ukv_read( // level_db_t& db = *reinterpret_cast(c_db); leveldb::ReadOptions options; strided_iterator_gt keys {c_keys, c_keys_stride}; - places_arg_t tasks {{}, keys, c_tasks_count}; + places_arg_t tasks {{}, keys, {}, c_tasks_count}; auto value_uptr = make_value(c_error); std::string& value = *value_uptr.get(); diff --git a/src/backend_rocksdb.cpp b/src/backend_rocksdb.cpp index 3e4ce9c4f..079b16adf 100644 --- a/src/backend_rocksdb.cpp +++ b/src/backend_rocksdb.cpp @@ -231,7 +231,7 @@ void ukv_write( // strided_iterator_gt vals {c_vals, c_vals_stride}; strided_iterator_gt offs {c_offs, c_offs_stride}; strided_iterator_gt lens {c_lens, c_lens_stride}; - contents_arg_t tasks {cols, keys, vals, offs, lens, c_tasks_count}; + contents_arg_t tasks {vals, offs, lens, {}, c_tasks_count}; rocksdb::WriteOptions options; if (c_options & ukv_option_write_flush_k) @@ -442,7 +442,7 @@ void ukv_read( // rocks_txn_t* txn = reinterpret_cast(c_txn); strided_iterator_gt cols_stride {c_cols, c_cols_stride}; strided_iterator_gt keys_stride {c_keys, c_keys_stride}; - places_arg_t tasks {cols_stride, keys_stride, c_tasks_count}; + places_arg_t tasks {cols_stride, keys_stride, {}, c_tasks_count}; stl_arena_t arena = clean_arena(c_arena, c_error); rocksdb::ReadOptions options; From 84a75274d05efd02bf862d96b715eb493a32550a Mon Sep 17 00:00:00 2001 From: violeta Date: Wed, 31 Aug 2022 10:34:55 +0400 Subject: [PATCH 09/30] Minor: code correction --- src/backend_leveldb.cpp | 2 +- src/backend_rocksdb.cpp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/backend_leveldb.cpp b/src/backend_leveldb.cpp index 6d2f64e42..d4f7fc8bb 100644 --- a/src/backend_leveldb.cpp +++ b/src/backend_leveldb.cpp @@ -129,7 +129,7 @@ void write_many( // for (std::size_t i = 0; i != tasks.size(); ++i) { auto task = tasks[i]; auto key = to_slice(task); - if (task) + if (!task) batch.Delete(key); else batch.Put(key, to_slice(task)); diff --git a/src/backend_rocksdb.cpp b/src/backend_rocksdb.cpp index 079b16adf..e694261b1 100644 --- a/src/backend_rocksdb.cpp +++ b/src/backend_rocksdb.cpp @@ -152,11 +152,11 @@ void write_one( // rocks_status_t status; if (txn) - status = task // + status = !task // ? txn->SingleDelete(col, key) : txn->Put(col, key, to_slice(task)); else - status = task // + status = !task // ? db.native->SingleDelete(options, col, key) : db.native->Put(options, col, key, to_slice(task)); @@ -187,7 +187,7 @@ void write_many( // auto task = tasks[i]; auto col = rocks_collection(db, task.col); auto key = to_slice(task); - auto status = task // + auto status = !task // ? batch.Delete(col, key) : batch.Put(col, key, to_slice(task)); export_error(status, c_error); From 6a5c972797366079eb577622fae1230a27c2ac9d Mon Sep 17 00:00:00 2001 From: violeta Date: Wed, 31 Aug 2022 10:48:29 +0400 Subject: [PATCH 10/30] Build fix: add ukv_col_drop(...) for arrow_client. --- src/arrow_client.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/arrow_client.cpp b/src/arrow_client.cpp index de545dede..490ea4ae2 100644 --- a/src/arrow_client.cpp +++ b/src/arrow_client.cpp @@ -565,6 +565,19 @@ void ukv_col_upsert( std::memcpy(c_col, id_ptr->body->data(), sizeof(ukv_col_t)); } +void ukv_col_drop( + // Inputs: + ukv_t const c_db, + ukv_str_view_t c_col_name, + ukv_col_t c_col_id, + ukv_col_drop_mode_t c_mode, + // Outputs: + ukv_error_t* c_error) { + + return_if_error(c_db, c_error, uninitialized_state_k, "DataBase is uninitialized"); + rpc_client_t& db = *reinterpret_cast(c_db); +} + void ukv_col_remove( // Inputs: ukv_t const c_db, From c3f112cb547e0984fca35162fc39f361b4387fe5 Mon Sep 17 00:00:00 2001 From: violeta Date: Wed, 31 Aug 2022 10:50:51 +0400 Subject: [PATCH 11/30] Remove ukv_col_remove(...) ukv. --- src/arrow_client.cpp | 11 ----------- src/backend_leveldb.cpp | 19 ------------------- src/backend_rocksdb.cpp | 30 ------------------------------ 3 files changed, 60 deletions(-) diff --git a/src/arrow_client.cpp b/src/arrow_client.cpp index 490ea4ae2..255289d20 100644 --- a/src/arrow_client.cpp +++ b/src/arrow_client.cpp @@ -578,17 +578,6 @@ void ukv_col_drop( rpc_client_t& db = *reinterpret_cast(c_db); } -void ukv_col_remove( - // Inputs: - ukv_t const c_db, - ukv_str_view_t c_col_name, - // Outputs: - ukv_error_t* c_error) { - - return_if_error(c_db, c_error, uninitialized_state_k, "DataBase is uninitialized"); - rpc_client_t& db = *reinterpret_cast(c_db); -} - void ukv_col_list( // ukv_t const c_db, ukv_size_t* c_count, diff --git a/src/backend_leveldb.cpp b/src/backend_leveldb.cpp index d4f7fc8bb..cff34d0bc 100644 --- a/src/backend_leveldb.cpp +++ b/src/backend_leveldb.cpp @@ -567,25 +567,6 @@ void ukv_col_drop( } } -void ukv_col_remove( // - ukv_t const c_db, - ukv_str_view_t c_col_name, - ukv_error_t* c_error) { - - return_if_error(c_db, c_error, uninitialized_state_k, "DataBase is uninitialized"); - - if (c_col_name && std::strlen(c_col_name) && (*c_error = "Collections not supported by LevelDB!")) - return; - - level_db_t& db = *reinterpret_cast(c_db); - leveldb::WriteBatch batch; - auto it = std::unique_ptr(db.NewIterator(leveldb::ReadOptions())); - for (it->SeekToFirst(); it->Valid(); it->Next()) - batch.Delete(it->key()); - level_status_t status = db.Write(leveldb::WriteOptions(), &batch); - export_error(status, c_error); -} - void ukv_col_list( // ukv_t const c_db, ukv_size_t* c_count, diff --git a/src/backend_rocksdb.cpp b/src/backend_rocksdb.cpp index e694261b1..18e4d368f 100644 --- a/src/backend_rocksdb.cpp +++ b/src/backend_rocksdb.cpp @@ -703,36 +703,6 @@ void ukv_col_drop( } } -void ukv_col_remove( // - ukv_t const c_db, - ukv_str_view_t c_col_name, - ukv_error_t* c_error) { - - return_if_error(c_db, c_error, uninitialized_state_k, "DataBase is uninitialized"); - - rocks_db_t& db = *reinterpret_cast(c_db); - if (!c_col_name || !std::strlen(c_col_name)) { - rocksdb::WriteBatch batch; - auto col = db.native->DefaultColumnFamily(); - auto it = std::unique_ptr(db.native->NewIterator(rocksdb::ReadOptions(), col)); - for (it->SeekToFirst(); it->Valid(); it->Next()) - batch.Delete(col, it->key()); - rocks_status_t status = db.native->Write(rocksdb::WriteOptions(), &batch); - export_error(status, c_error); - return; - } - - for (auto it = db.columns.begin(); it != db.columns.end(); it++) { - if (c_col_name == (*it)->GetName() && (*it)->GetName() != "default") { - rocks_status_t status = db.native->DropColumnFamily(*it); - if (export_error(status, c_error)) - return; - db.columns.erase(it--); - break; - } - } -} - void ukv_col_list( // ukv_t const c_db, ukv_size_t* c_count, From d258d4ab31f8c399b611aaf13ccfa00ce1e7f563 Mon Sep 17 00:00:00 2001 From: violeta Date: Wed, 31 Aug 2022 11:29:44 +0400 Subject: [PATCH 12/30] Correct usage options in leveldb. --- src/backend_leveldb.cpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/backend_leveldb.cpp b/src/backend_leveldb.cpp index cff34d0bc..c2cf0b2c3 100644 --- a/src/backend_leveldb.cpp +++ b/src/backend_leveldb.cpp @@ -360,11 +360,11 @@ void ukv_read( // try { if (c_tasks_count == 1) { - auto func = (c_options /* & ukv_option_read_lengths_k */) ? &measure_one : &read_one; + auto func = c_options ? &measure_one : &read_one; func(db, tasks, options, value, c_found_values, c_found_offsets, c_found_lengths, arena, c_error); } else { - auto func = (c_options /* & ukv_option_read_lengths_k */) ? &measure_many : &read_many; + auto func = c_options ? &measure_many : &read_many; func(db, tasks, options, value, c_found_values, c_found_offsets, c_found_lengths, arena, c_error); } } @@ -407,13 +407,12 @@ void ukv_scan( // strided_iterator_gt lens {c_scan_lengths, c_scan_lengths_stride}; scans_arg_t tasks {{}, keys, lens, c_min_tasks_count}; - bool export_lengths = (c_options /* & ukv_option_read_lengths_k */); leveldb::ReadOptions options; options.fill_cache = false; ukv_size_t total_lengths = reduce_n(tasks.lengths, tasks.count, 0ul); ukv_size_t total_bytes = total_lengths * sizeof(ukv_key_t); - if (export_lengths) + if (c_options) total_bytes += total_lengths * sizeof(ukv_val_len_t); span_gt tape = arena.alloc(total_bytes, c_error); @@ -422,7 +421,7 @@ void ukv_scan( // ukv_key_t* found_keys = reinterpret_cast(tape.begin()); ukv_val_len_t* found_lens = reinterpret_cast(found_keys + total_lengths); *c_found_keys = found_keys; - *c_found_lengths = export_lengths ? found_lens : nullptr; + *c_found_lengths = c_options ? found_lens : nullptr; level_iter_uptr_t it; try { @@ -439,13 +438,13 @@ void ukv_scan( // ukv_size_t j = 0; for (; it->Valid() && j != task.length; j++, it->Next()) { std::memcpy(&found_keys[j], it->key().data(), sizeof(ukv_key_t)); - if (export_lengths) + if (c_options) found_lens[j] = static_cast(it->value().size()); } while (j != task.length) { found_keys[j] = ukv_key_unknown_k; - if (export_lengths) + if (c_options) found_lens[j] = ukv_val_len_missing_k; ++j; } From 8ceb98e250f7c9872704db0bd658d4d1969ccdf3 Mon Sep 17 00:00:00 2001 From: violeta Date: Wed, 31 Aug 2022 11:48:36 +0400 Subject: [PATCH 13/30] Build fix; refactoring LevelDB --- src/backend_leveldb.cpp | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/src/backend_leveldb.cpp b/src/backend_leveldb.cpp index c2cf0b2c3..23a9a35a8 100644 --- a/src/backend_leveldb.cpp +++ b/src/backend_leveldb.cpp @@ -109,30 +109,35 @@ void ukv_db_open(ukv_str_view_t, ukv_t* c_db, ukv_error_t* c_error) { void write_one( // level_db_t& db, - contents_arg_t const& tasks, + places_arg_t places, + contents_arg_t const& contents, leveldb::WriteOptions const& options, ukv_error_t* c_error) { - auto task = tasks[0]; - auto key = to_slice(task); - level_status_t status = task ? db.Delete(options, key) : db.Put(options, key, to_slice(task)); + auto place = places[0]; + auto content = contents[0]; + auto key = to_slice(place.key); + level_status_t status = content ? db.Delete(options, key) : db.Put(options, key, to_slice(content)); export_error(status, c_error); } void write_many( // level_db_t& db, - contents_arg_t const& tasks, + places_arg_t places, + contents_arg_t const& contents, leveldb::WriteOptions const& options, ukv_error_t* c_error) { leveldb::WriteBatch batch; - for (std::size_t i = 0; i != tasks.size(); ++i) { - auto task = tasks[i]; - auto key = to_slice(task); - if (!task) + for (std::size_t i = 0; i != places.size(); ++i) { + auto place = places[i]; + auto content = contents[i]; + + auto key = to_slice(place.key); + if (!content) batch.Delete(key); else - batch.Put(key, to_slice(task)); + batch.Put(key, to_slice(content)); } level_status_t status = db.Write(options, &batch); @@ -159,14 +164,13 @@ void ukv_write( // ukv_val_len_t const* c_lens, ukv_size_t const c_lens_stride, + ukv_1x8_t const* c_presences, + ukv_options_t const c_options, ukv_arena_t*, ukv_error_t* c_error) { - if (!c_db) { - *c_error = "DataBase is NULL!"; - return; - } + return_if_error(c_db, c_error, uninitialized_state_k, "DataBase is uninitialized"); level_db_t& db = *reinterpret_cast(c_db); strided_iterator_gt cols {c_cols, c_cols_stride}; @@ -174,7 +178,10 @@ void ukv_write( // strided_iterator_gt vals {c_vals, c_vals_stride}; strided_iterator_gt offs {c_offs, c_offs_stride}; strided_iterator_gt lens {c_lens, c_lens_stride}; - contents_arg_t tasks {vals, offs, lens, {}, c_tasks_count}; + strided_iterator_gt presences {c_presences, sizeof(ukv_1x8_t)}; + + places_arg_t places {cols, keys, {}, c_tasks_count}; + contents_arg_t contents {vals, offs, lens, presences, c_tasks_count}; leveldb::WriteOptions options; if (c_options & ukv_option_write_flush_k) @@ -182,7 +189,7 @@ void ukv_write( // try { auto func = c_tasks_count == 1 ? &write_one : &write_many; - func(db, tasks, options, c_error); + func(db, places, contents, options, c_error); } catch (...) { *c_error = "Write Failure"; From cf9378beefb86131b116892a6ed325ed06113abc Mon Sep 17 00:00:00 2001 From: violeta Date: Wed, 31 Aug 2022 11:53:58 +0400 Subject: [PATCH 14/30] Minor --- src/backend_leveldb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend_leveldb.cpp b/src/backend_leveldb.cpp index 23a9a35a8..d3cfc306e 100644 --- a/src/backend_leveldb.cpp +++ b/src/backend_leveldb.cpp @@ -117,7 +117,7 @@ void write_one( // auto place = places[0]; auto content = contents[0]; auto key = to_slice(place.key); - level_status_t status = content ? db.Delete(options, key) : db.Put(options, key, to_slice(content)); + level_status_t status = !content ? db.Delete(options, key) : db.Put(options, key, to_slice(content)); export_error(status, c_error); } From 065768932d1b01b55cbac946af7261b5be60eff9 Mon Sep 17 00:00:00 2001 From: violeta Date: Wed, 31 Aug 2022 13:38:33 +0400 Subject: [PATCH 15/30] Update: code correction. --- src/backend_leveldb.cpp | 45 +++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/src/backend_leveldb.cpp b/src/backend_leveldb.cpp index d3cfc306e..76aa54920 100644 --- a/src/backend_leveldb.cpp +++ b/src/backend_leveldb.cpp @@ -109,7 +109,7 @@ void ukv_db_open(ukv_str_view_t, ukv_t* c_db, ukv_error_t* c_error) { void write_one( // level_db_t& db, - places_arg_t places, + places_arg_t const& places, contents_arg_t const& contents, leveldb::WriteOptions const& options, ukv_error_t* c_error) { @@ -123,7 +123,7 @@ void write_one( // void write_many( // level_db_t& db, - places_arg_t places, + places_arg_t const& places, contents_arg_t const& contents, leveldb::WriteOptions const& options, ukv_error_t* c_error) { @@ -287,7 +287,7 @@ void measure_many( // void read_many( // level_db_t& db, - places_arg_t const& tasks, + places_arg_t const& places, leveldb::ReadOptions const& options, std::string& value, ukv_val_ptr_t* c_found_values, @@ -296,18 +296,18 @@ void read_many( // stl_arena_t& arena, ukv_error_t* c_error) { - ukv_size_t lens_bytes = sizeof(ukv_val_len_t) * tasks.count; + ukv_size_t lens_bytes = sizeof(ukv_val_len_t) * places.count; span_gt tape = arena.alloc(lens_bytes * 2, c_error); return_on_error(c_error); ukv_val_len_t* lens = reinterpret_cast(tape.begin()); - ukv_val_len_t* offs = lens + tasks.count; - ukv_val_ptr_t contents = reinterpret_cast(offs + tasks.count); - std::fill_n(lens, tasks.count * 2, ukv_val_len_missing_k); + ukv_val_len_t* offs = lens + places.count; + ukv_val_ptr_t contents = reinterpret_cast(offs + places.count); + std::fill_n(lens, places.count * 2, ukv_val_len_missing_k); - for (std::size_t i = 0; i != tasks.size(); ++i) { - place_t task = tasks[i]; - level_status_t status = db.Get(options, to_slice(task.key), &value); + for (std::size_t i = 0; i != places.size(); ++i) { + place_t place = places[i]; + level_status_t status = db.Get(options, to_slice(place.key), &value); if (status.IsNotFound()) continue; if (export_error(status, c_error)) @@ -319,8 +319,8 @@ void read_many( // return_on_error(c_error); lens = reinterpret_cast(tape.begin()); - offs = lens + tasks.count; - contents = reinterpret_cast(offs + tasks.count); + offs = lens + places.count; + contents = reinterpret_cast(offs + places.count); std::memcpy(tape.begin() + old_tape_len, value.data(), bytes_in_value); lens[i] = static_cast(bytes_in_value); @@ -348,6 +348,7 @@ void ukv_read( // ukv_val_ptr_t* c_found_values, ukv_val_len_t** c_found_offsets, ukv_val_len_t** c_found_lengths, + ukv_1x8_t** c_found_presences, ukv_arena_t* c_arena, ukv_error_t* c_error) { @@ -360,7 +361,7 @@ void ukv_read( // level_db_t& db = *reinterpret_cast(c_db); leveldb::ReadOptions options; strided_iterator_gt keys {c_keys, c_keys_stride}; - places_arg_t tasks {{}, keys, {}, c_tasks_count}; + places_arg_t places {{}, keys, {}, c_tasks_count}; auto value_uptr = make_value(c_error); std::string& value = *value_uptr.get(); @@ -368,11 +369,11 @@ void ukv_read( // try { if (c_tasks_count == 1) { auto func = c_options ? &measure_one : &read_one; - func(db, tasks, options, value, c_found_values, c_found_offsets, c_found_lengths, arena, c_error); + func(db, places, options, value, c_found_values, c_found_offsets, c_found_lengths, arena, c_error); } else { auto func = c_options ? &measure_many : &read_many; - func(db, tasks, options, value, c_found_values, c_found_offsets, c_found_lengths, arena, c_error); + func(db, places, options, value, c_found_values, c_found_offsets, c_found_lengths, arena, c_error); } } catch (...) { @@ -404,8 +405,6 @@ void ukv_scan( // return_if_error(c_db, c_error, uninitialized_state_k, "DataBase is uninitialized"); - return_if_error(c_db, c_error, uninitialized_state_k, "DataBase is uninitialized"); - stl_arena_t arena = clean_arena(c_arena, c_error); return_on_error(c_error); @@ -520,6 +519,10 @@ void ukv_size( // } } +/*********************************************************/ +/***************** Collections Management ****************/ +/*********************************************************/ + void ukv_col_upsert( // ukv_t const, ukv_str_view_t c_col_name, @@ -602,6 +605,10 @@ void ukv_db_control( // *c_error = "Controls aren't supported in this implementation!"; } +/*********************************************************/ +/***************** Transactions ****************/ +/*********************************************************/ + void ukv_txn_begin( // ukv_t const, ukv_size_t const, @@ -618,6 +625,10 @@ void ukv_txn_commit( // *c_error = "Transactions not supported by LevelDB!"; } +/*********************************************************/ +/***************** Memory Management ****************/ +/*********************************************************/ + void ukv_arena_free(ukv_t const, ukv_arena_t c_arena) { if (!c_arena) return; From 053c08aa3733335d3b1a82cb8a9e1b777c6b0b0f Mon Sep 17 00:00:00 2001 From: violeta Date: Thu, 1 Sep 2022 12:34:12 +0400 Subject: [PATCH 16/30] Update : Correct has_one_binary(...) --- python/pybind/crud.hpp | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/python/pybind/crud.hpp b/python/pybind/crud.hpp index e315be885..e3a3cc182 100644 --- a/python/pybind/crud.hpp +++ b/python/pybind/crud.hpp @@ -137,10 +137,7 @@ static py::object has_one_binary(py_col_t& col, PyObject* key_py) { status_t status; ukv_key_t key = py_to_scalar(key_py); - ukv_val_ptr_t found_values = nullptr; - ukv_val_len_t* found_offsets = nullptr; - ukv_val_len_t* found_lengths = nullptr; - auto options = static_cast(col.options() /* | ukv_option_read_lengths_k */); + ukv_1x8_t* presenses = nullptr; { [[maybe_unused]] py::gil_scoped_release release; @@ -151,16 +148,18 @@ static py::object has_one_binary(py_col_t& col, PyObject* key_py) { 0, &key, 0, - options, - &found_values, - &found_offsets, - &found_lengths, + col.options(), + nullptr, + nullptr, + nullptr, + &presenses, col.member_arena(), status.member_ptr()); status.throw_unhandled(); } - PyObject* obj_ptr = *found_lengths != ukv_val_len_missing_k ? Py_True : Py_False; + strided_iterator_gt presenses_it {presenses, sizeof(ukv_1x8_t)}; + PyObject* obj_ptr = presenses_it[0] ? Py_True : Py_False; return py::reinterpret_borrow(obj_ptr); } From 5d2be7caaf0eeefb0f2d027be3327428b01afa36 Mon Sep 17 00:00:00 2001 From: violeta Date: Thu, 1 Sep 2022 17:10:25 +0400 Subject: [PATCH 17/30] Minor --- src/backend_stl.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backend_stl.cpp b/src/backend_stl.cpp index b3c076d59..90d09aa98 100644 --- a/src/backend_stl.cpp +++ b/src/backend_stl.cpp @@ -553,7 +553,6 @@ void ukv_read( // // 3. Pull the data, once we know the total length ukv_val_len_t progress_in_tape = 0; - ukv_val_len_t last_value_length = 0; auto tape = arena.alloc(total_length, c_error); auto data_enumerator = [&](std::size_t i, value_view_t value) { offs[i] = progress_in_tape; From 0ef3eef1d06b9d45f695ffecfdb9dd6a0c04ba41 Mon Sep 17 00:00:00 2001 From: violeta Date: Fri, 2 Sep 2022 12:31:55 +0400 Subject: [PATCH 18/30] Default arena type to growing. Fix: minor --- src/helpers.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/helpers.hpp b/src/helpers.hpp index 4bedb0c68..1d2743f45 100644 --- a/src/helpers.hpp +++ b/src/helpers.hpp @@ -259,10 +259,10 @@ class monotonic_resource_t final : public std::pmr::memory_resource { buffer_t buf = buffers_.front(); buffers_.pop_front(); while (!buffers_.empty()) { - buffers_.pop_front(); release_one(buf); upstream_->deallocate(buf.begin, buf.total_memory, alignment_); buf = buffers_.front(); + buffers_.pop_front(); } buffers_.push_front(buf); @@ -373,7 +373,7 @@ struct stl_arena_t { : resource(mem_resource), using_shared_memory(false) {} explicit stl_arena_t( // std::size_t initial_buffer_size = 1024ul * 1024ul, - monotonic_resource_t::type_t type = monotonic_resource_t::capped_k, + monotonic_resource_t::type_t type = monotonic_resource_t::growing_k, bool use_shared_memory = false) : resource(initial_buffer_size, 64ul, @@ -444,7 +444,7 @@ inline stl_arena_t prepare_arena(ukv_arena_t* c_arena, ukv_options_t options, uk if (!*arena || ((options & ukv_option_read_shared_k) && !(*arena)->using_shared_memory)) { delete *arena; *arena = - new stl_arena_t(1024ul * 1024ul, monotonic_resource_t::capped_k, options & ukv_option_read_shared_k); + new stl_arena_t(1024ul * 1024ul, monotonic_resource_t::growing_k, options & ukv_option_read_shared_k); } if (!(options & ukv_option_nodiscard_k)) @@ -499,7 +499,7 @@ class file_handle_t { std::fclose(handle_); } - operator std::FILE*() const noexcept { return handle_; } + operator std::FILE *() const noexcept { return handle_; } }; template From fe8c684e0b961785b4700e5af656fea1d16396dc Mon Sep 17 00:00:00 2001 From: violeta Date: Fri, 2 Sep 2022 13:13:22 +0400 Subject: [PATCH 19/30] Make delete monotonic_resource_t copy and ass. --- src/helpers.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/helpers.hpp b/src/helpers.hpp index 1d2743f45..fa8a3000b 100644 --- a/src/helpers.hpp +++ b/src/helpers.hpp @@ -234,6 +234,9 @@ class monotonic_resource_t final : public std::pmr::memory_resource { type_t type_; public: + monotonic_resource_t(monotonic_resource_t const&) = delete; + monotonic_resource_t& operator=(monotonic_resource_t const&) = delete; + explicit monotonic_resource_t(monotonic_resource_t* upstream) noexcept : buffers_(), upstream_(upstream), alignment_(upstream->alignment_), type_(type_t::borrowed_k) {}; From aa0ebffc3f7ca1b83de934bb21eae5e3a33c10e6 Mon Sep 17 00:00:00 2001 From: violeta Date: Fri, 2 Sep 2022 13:34:31 +0400 Subject: [PATCH 20/30] Correct usage options in rocksdb. --- src/backend_rocksdb.cpp | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/backend_rocksdb.cpp b/src/backend_rocksdb.cpp index d9c9cd10e..452fb1541 100644 --- a/src/backend_rocksdb.cpp +++ b/src/backend_rocksdb.cpp @@ -445,18 +445,18 @@ void ukv_read( // places_arg_t tasks {cols_stride, keys_stride, {}, c_tasks_count}; stl_arena_t arena = prepare_arena(c_arena, {}, c_error); - + rocksdb::ReadOptions options; if (txn && (c_options & ukv_option_txn_snapshot_k)) options.snapshot = txn->GetSnapshot(); try { if (c_tasks_count == 1) { - auto func = (c_options /* & ukv_option_read_lengths_k) */) ? &measure_one : &read_one; + auto func = c_options ? &measure_one : &read_one; func(db, txn, tasks, options, c_found_values, c_found_offsets, c_found_lengths, arena, c_error); } else { - auto func = (c_options /* & ukv_option_read_lengths_k */) ? &measure_many : &read_many; + auto func = c_options ? &measure_many : &read_many; func(db, txn, tasks, options, c_found_values, c_found_offsets, c_found_lengths, arena, c_error); } } @@ -502,13 +502,12 @@ void ukv_scan( // strided_iterator_gt lengths {c_scan_lengths, c_scan_lengths_stride}; scans_arg_t tasks {cols, keys, lengths, c_min_tasks_count}; - bool export_lengths = (c_options /* & ukv_option_read_lengths_k */); rocksdb::ReadOptions options; options.fill_cache = false; ukv_size_t total_lengths = reduce_n(tasks.lengths, tasks.count, 0ul); ukv_size_t total_bytes = total_lengths * sizeof(ukv_key_t); - if (export_lengths) + if (c_options) total_bytes += total_lengths * sizeof(ukv_val_len_t); span_gt tape = arena.alloc(total_bytes, c_error); @@ -517,7 +516,7 @@ void ukv_scan( // ukv_key_t* found_keys = reinterpret_cast(tape.begin()); ukv_val_len_t* found_lens = reinterpret_cast(found_keys + total_lengths); *c_found_keys = found_keys; - *c_found_lengths = export_lengths ? found_lens : nullptr; + *c_found_lengths = c_options ? found_lens : nullptr; for (ukv_size_t i = 0; i != c_min_tasks_count; ++i) { scan_t task = tasks[i]; @@ -536,19 +535,19 @@ void ukv_scan( // it->Seek(to_slice(task.min_key)); for (; it->Valid() && j != task.length; j++, it->Next()) { std::memcpy(&found_keys[j], it->key().data(), sizeof(ukv_key_t)); - if (export_lengths) + if (c_options) found_lens[j] = static_cast(it->value().size()); } while (j != task.length) { found_keys[j] = ukv_key_unknown_k; - if (export_lengths) + if (c_options) found_lens[j] = ukv_val_len_missing_k; ++j; } found_keys += task.length; - if (export_lengths) + if (c_options) found_lens += task.length; } } From a3485d0fb3060d4db0d2f5ed9434f9c0956876c5 Mon Sep 17 00:00:00 2001 From: violeta Date: Fri, 2 Sep 2022 13:34:42 +0400 Subject: [PATCH 21/30] Minor --- src/test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test.cpp b/src/test.cpp index 85fd1c04a..384ef0852 100644 --- a/src/test.cpp +++ b/src/test.cpp @@ -746,7 +746,7 @@ TEST(db, remove_vertexes) { for (size_t vertex_id = 0; vertex_id < vertexes_count; ++vertex_id) { EXPECT_TRUE(*graph.contains(vertex_id)); EXPECT_TRUE(graph.remove(vertex_id)); - EXPECT_FALSE(*graph.contains(vertex_id)); + EXPECT_FALSE(graph.contains(vertex_id)); } } From 753f9602c1a8251d6fc9229c34c312b5f52a2bc0 Mon Sep 17 00:00:00 2001 From: violeta Date: Fri, 2 Sep 2022 13:38:44 +0400 Subject: [PATCH 22/30] Build fix: include correction --- src/helpers.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/helpers.hpp b/src/helpers.hpp index fa8a3000b..bf6ce3396 100644 --- a/src/helpers.hpp +++ b/src/helpers.hpp @@ -12,6 +12,7 @@ #include // `std::allocator` #include // `std::vector` #include // `std::sort` +#include // `std::accumulate` #include #if __APPLE__ From 04e0fb2d5fa55fe1b218308e79f2d4708dd856b3 Mon Sep 17 00:00:00 2001 From: violeta Date: Fri, 2 Sep 2022 13:58:41 +0400 Subject: [PATCH 23/30] Build fix: refactoring rocksdb. --- src/backend_rocksdb.cpp | 56 ++++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/src/backend_rocksdb.cpp b/src/backend_rocksdb.cpp index 452fb1541..64ebf931a 100644 --- a/src/backend_rocksdb.cpp +++ b/src/backend_rocksdb.cpp @@ -142,23 +142,25 @@ void ukv_db_open(ukv_str_view_t, ukv_t* c_db, ukv_error_t* c_error) { void write_one( // rocks_db_t& db, rocks_txn_t* txn, - contents_arg_t const& tasks, + places_arg_t const& places, + contents_arg_t const& contents, rocksdb::WriteOptions const& options, ukv_error_t* c_error) { - auto task = tasks[0]; - auto col = rocks_collection(db, task.col); - auto key = to_slice(task); + auto place = places[0]; + auto content = contents[0]; + auto col = rocks_collection(db, place.col); + auto key = to_slice(place.key); rocks_status_t status; if (txn) - status = !task // + status = !content // ? txn->SingleDelete(col, key) - : txn->Put(col, key, to_slice(task)); + : txn->Put(col, key, to_slice(content)); else - status = !task // + status = !content // ? db.native->SingleDelete(options, col, key) - : db.native->Put(options, col, key, to_slice(task)); + : db.native->Put(options, col, key, to_slice(content)); export_error(status, c_error); } @@ -166,30 +168,33 @@ void write_one( // void write_many( // rocks_db_t& db, rocks_txn_t* txn, - contents_arg_t const& tasks, + places_arg_t const& places, + contents_arg_t const& contents, rocksdb::WriteOptions const& options, ukv_error_t* c_error) { if (txn) { - for (std::size_t i = 0; i != tasks.size(); ++i) { - auto task = tasks[i]; - auto col = rocks_collection(db, task.col); - auto key = to_slice(task); - auto status = task // + for (std::size_t i = 0; i != places.size(); ++i) { + auto place = places[i]; + auto content = contents[i]; + auto col = rocks_collection(db, place.col); + auto key = to_slice(place.col); + auto status = !content // ? txn->Delete(col, key) - : txn->Put(col, key, to_slice(task)); + : txn->Put(col, key, to_slice(content)); export_error(status, c_error); } } else { rocksdb::WriteBatch batch; - for (std::size_t i = 0; i != tasks.size(); ++i) { - auto task = tasks[i]; - auto col = rocks_collection(db, task.col); - auto key = to_slice(task); - auto status = !task // + for (std::size_t i = 0; i != places.size(); ++i) { + auto place = places[i]; + auto content = contents[i]; + auto col = rocks_collection(db, place.col); + auto key = to_slice(place.key); + auto status = !content // ? batch.Delete(col, key) - : batch.Put(col, key, to_slice(task)); + : batch.Put(col, key, to_slice(content)); export_error(status, c_error); } @@ -218,6 +223,8 @@ void ukv_write( // ukv_val_len_t const* c_lens, ukv_size_t const c_lens_stride, + ukv_1x8_t const* c_presences, + ukv_options_t const c_options, ukv_arena_t*, ukv_error_t* c_error) { @@ -231,7 +238,10 @@ void ukv_write( // strided_iterator_gt vals {c_vals, c_vals_stride}; strided_iterator_gt offs {c_offs, c_offs_stride}; strided_iterator_gt lens {c_lens, c_lens_stride}; - contents_arg_t tasks {vals, offs, lens, {}, c_tasks_count}; + strided_iterator_gt presences {c_presences, sizeof(ukv_1x8_t)}; + + places_arg_t places {cols, keys, {}, c_tasks_count}; + contents_arg_t contents {vals, offs, lens, presences, c_tasks_count}; rocksdb::WriteOptions options; if (c_options & ukv_option_write_flush_k) @@ -239,7 +249,7 @@ void ukv_write( // try { auto func = c_tasks_count == 1 ? &write_one : &write_many; - func(db, txn, tasks, options, c_error); + func(db, txn, places, contents, options, c_error); } catch (...) { *c_error = "Write Failure"; From 4b4b65b305605a9e5e645ed6e310f366df946a90 Mon Sep 17 00:00:00 2001 From: violeta Date: Fri, 2 Sep 2022 15:18:38 +0400 Subject: [PATCH 24/30] rocksdb interface correction --- src/backend_rocksdb.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backend_rocksdb.cpp b/src/backend_rocksdb.cpp index 64ebf931a..fd9f00aaf 100644 --- a/src/backend_rocksdb.cpp +++ b/src/backend_rocksdb.cpp @@ -437,6 +437,7 @@ void ukv_read( // ukv_val_ptr_t* c_found_values, ukv_val_len_t** c_found_offsets, ukv_val_len_t** c_found_lengths, + ukv_1x8_t** c_found_presences, ukv_arena_t* c_arena, ukv_error_t* c_error) { From e76f151da9ddbf72d6645615a8c7d8b91a0c4bcd Mon Sep 17 00:00:00 2001 From: violeta Date: Mon, 5 Sep 2022 11:19:08 +0400 Subject: [PATCH 25/30] Interface correction ukv_scan in leveldb and rocksdb. --- src/backend_leveldb.cpp | 2 +- src/backend_rocksdb.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend_leveldb.cpp b/src/backend_leveldb.cpp index b55010a2c..23711671d 100644 --- a/src/backend_leveldb.cpp +++ b/src/backend_leveldb.cpp @@ -397,7 +397,7 @@ void ukv_scan( // ukv_options_t const c_options, - ukv_key_t** c_found_keys, + ukv_key_t*** c_found_keys, ukv_val_len_t** c_found_lengths, ukv_arena_t* c_arena, diff --git a/src/backend_rocksdb.cpp b/src/backend_rocksdb.cpp index fd9f00aaf..f348a8b17 100644 --- a/src/backend_rocksdb.cpp +++ b/src/backend_rocksdb.cpp @@ -492,7 +492,7 @@ void ukv_scan( // ukv_options_t const c_options, - ukv_key_t** c_found_keys, + ukv_key_t*** c_found_keys, ukv_val_len_t** c_found_lengths, ukv_arena_t* c_arena, From a23eac4fac5a0fedbffe5528b1bd57a374cd8ce8 Mon Sep 17 00:00:00 2001 From: violeta Date: Mon, 5 Sep 2022 12:24:56 +0400 Subject: [PATCH 26/30] Minor --- src/test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test.cpp b/src/test.cpp index 384ef0852..2b9bec045 100644 --- a/src/test.cpp +++ b/src/test.cpp @@ -729,7 +729,7 @@ TEST(db, upsert_edges) { fill_edges(); EXPECT_TRUE(graph.upsert(edges(es))); for (size_t vertex_id = 0; vertex_id < vertexes_count; ++vertex_id) { - EXPECT_TRUE(*graph.contains(vertex_id)); + EXPECT_TRUE(graph.contains(vertex_id)); EXPECT_EQ(*graph.degree(vertex_id), 9u); } } @@ -744,7 +744,7 @@ TEST(db, remove_vertexes) { EXPECT_TRUE(graph.upsert(edges(es))); for (size_t vertex_id = 0; vertex_id < vertexes_count; ++vertex_id) { - EXPECT_TRUE(*graph.contains(vertex_id)); + EXPECT_TRUE(graph.contains(vertex_id)); EXPECT_TRUE(graph.remove(vertex_id)); EXPECT_FALSE(graph.contains(vertex_id)); } From 54c0301328313ae1b719ba05d7539a69410323a8 Mon Sep 17 00:00:00 2001 From: violeta Date: Mon, 5 Sep 2022 12:54:57 +0400 Subject: [PATCH 27/30] Build fix: refactoring ukv interface in rocksdb and leveldb. --- src/backend_leveldb.cpp | 13 +++++++------ src/backend_rocksdb.cpp | 13 +++++++------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/backend_leveldb.cpp b/src/backend_leveldb.cpp index 23711671d..3d83c60f7 100644 --- a/src/backend_leveldb.cpp +++ b/src/backend_leveldb.cpp @@ -392,13 +392,14 @@ void ukv_scan( // ukv_key_t const* c_start_keys, ukv_size_t const c_start_keys_stride, - ukv_size_t const* c_scan_lengths, - ukv_size_t const c_scan_lengths_stride, + ukv_val_len_t const* c_scan_lengths, + ukv_val_len_t const c_scan_lengths_stride, ukv_options_t const c_options, - ukv_key_t*** c_found_keys, - ukv_val_len_t** c_found_lengths, + ukv_val_len_t** c_found_offsets, + ukv_val_len_t** c_found_counts, + ukv_key_t** c_found_keys, ukv_arena_t* c_arena, ukv_error_t* c_error) { @@ -411,7 +412,7 @@ void ukv_scan( // level_db_t& db = *reinterpret_cast(c_db); strided_iterator_gt keys {c_start_keys, c_start_keys_stride}; - strided_iterator_gt lens {c_scan_lengths, c_scan_lengths_stride}; + strided_iterator_gt lens {c_scan_lengths, c_scan_lengths_stride}; scans_arg_t tasks {{}, keys, lens, c_min_tasks_count}; leveldb::ReadOptions options; @@ -428,7 +429,7 @@ void ukv_scan( // ukv_key_t* found_keys = reinterpret_cast(tape.begin()); ukv_val_len_t* found_lens = reinterpret_cast(found_keys + total_lengths); *c_found_keys = found_keys; - *c_found_lengths = c_options ? found_lens : nullptr; + *c_found_counts = c_options ? found_lens : nullptr; level_iter_uptr_t it; try { diff --git a/src/backend_rocksdb.cpp b/src/backend_rocksdb.cpp index f348a8b17..3c61f7924 100644 --- a/src/backend_rocksdb.cpp +++ b/src/backend_rocksdb.cpp @@ -487,13 +487,14 @@ void ukv_scan( // ukv_key_t const* c_start_keys, ukv_size_t const c_start_keys_stride, - ukv_size_t const* c_scan_lengths, - ukv_size_t const c_scan_lengths_stride, + ukv_val_len_t const* c_scan_lengths, + ukv_val_len_t const c_scan_lengths_stride, ukv_options_t const c_options, - ukv_key_t*** c_found_keys, - ukv_val_len_t** c_found_lengths, + ukv_val_len_t** c_found_offsets, + ukv_val_len_t** c_found_counts, + ukv_key_t** c_found_keys, ukv_arena_t* c_arena, ukv_error_t* c_error) { @@ -510,7 +511,7 @@ void ukv_scan( // rocks_txn_t* txn = reinterpret_cast(c_txn); strided_iterator_gt cols {c_cols, c_cols_stride}; strided_iterator_gt keys {c_start_keys, c_start_keys_stride}; - strided_iterator_gt lengths {c_scan_lengths, c_scan_lengths_stride}; + strided_iterator_gt lengths {c_scan_lengths, c_scan_lengths_stride}; scans_arg_t tasks {cols, keys, lengths, c_min_tasks_count}; rocksdb::ReadOptions options; @@ -527,7 +528,7 @@ void ukv_scan( // ukv_key_t* found_keys = reinterpret_cast(tape.begin()); ukv_val_len_t* found_lens = reinterpret_cast(found_keys + total_lengths); *c_found_keys = found_keys; - *c_found_lengths = c_options ? found_lens : nullptr; + *c_found_counts = c_options ? found_lens : nullptr; for (ukv_size_t i = 0; i != c_min_tasks_count; ++i) { scan_t task = tasks[i]; From b08fa54173add13aebd18aeb59ea0d061efce56f Mon Sep 17 00:00:00 2001 From: violeta Date: Mon, 5 Sep 2022 13:02:42 +0400 Subject: [PATCH 28/30] Build fix --- .vscode/launch.json | 4 ++-- .vscode/tasks.json | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 270107586..96c623efc 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -36,7 +36,7 @@ "showDisplayString": true, "stopAtEntry": false, "externalConsole": false, - "preLaunchTask": "Build Native: Debug", + "preLaunchTask": "Build Native: Debug RocksDB Test", "MIMode": "gdb", "miDebuggerPath": "/usr/bin/gdb" }, @@ -54,7 +54,7 @@ "showDisplayString": true, "stopAtEntry": false, "externalConsole": false, - "preLaunchTask": "Build Native: Debug", + "preLaunchTask": "Build Native: Debug LevelDB Test", "MIMode": "gdb", "miDebuggerPath": "/usr/bin/gdb" }, diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 818a43b42..3a32f701d 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -7,6 +7,18 @@ "args": [], "type": "shell" }, + { + "label": "Build Native: Debug RocksDB Test", + "command": "cmake -DCMAKE_BUILD_TYPE=Debug . && make ukv_rocksdb_test", + "args": [], + "type": "shell" + }, + { + "label": "Build Native: Debug LevelDB Test", + "command": "cmake -DCMAKE_BUILD_TYPE=Debug . && make ukv_leveldb_test", + "args": [], + "type": "shell" + }, { "label": "Build Native: Debug Arrow Test", // Both must pass builds From 0264f7d18cb0ee407f81e90dc3b5ddece0bfd6c9 Mon Sep 17 00:00:00 2001 From: violeta Date: Mon, 5 Sep 2022 15:20:18 +0400 Subject: [PATCH 29/30] Refactoring: Implement ukv_scan(...) for backend_leveldb. #43 --- src/backend_leveldb.cpp | 39 ++++++++++++++++----------------------- src/backend_rocksdb.cpp | 2 +- 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/src/backend_leveldb.cpp b/src/backend_leveldb.cpp index 3d83c60f7..49734d314 100644 --- a/src/backend_leveldb.cpp +++ b/src/backend_leveldb.cpp @@ -393,7 +393,7 @@ void ukv_scan( // ukv_size_t const c_start_keys_stride, ukv_val_len_t const* c_scan_lengths, - ukv_val_len_t const c_scan_lengths_stride, + ukv_size_t const c_scan_lengths_stride, ukv_options_t const c_options, @@ -407,7 +407,6 @@ void ukv_scan( // return_if_error(c_db, c_error, uninitialized_state_k, "DataBase is uninitialized"); stl_arena_t arena = prepare_arena(c_arena, {}, c_error); - return_on_error(c_error); level_db_t& db = *reinterpret_cast(c_db); @@ -418,19 +417,17 @@ void ukv_scan( // leveldb::ReadOptions options; options.fill_cache = false; - ukv_size_t total_lengths = reduce_n(tasks.lengths, tasks.count, 0ul); - ukv_size_t total_bytes = total_lengths * sizeof(ukv_key_t); - if (c_options) - total_bytes += total_lengths * sizeof(ukv_val_len_t); - - span_gt tape = arena.alloc(total_bytes, c_error); + // 1. Allocate a tape for all the values to be fetched + auto offsets = arena.alloc_or_dummy(tasks.count + 1, c_error, c_found_offsets); + return_on_error(c_error); + auto counts = arena.alloc_or_dummy(tasks.count, c_error, c_found_counts); return_on_error(c_error); - ukv_key_t* found_keys = reinterpret_cast(tape.begin()); - ukv_val_len_t* found_lens = reinterpret_cast(found_keys + total_lengths); - *c_found_keys = found_keys; - *c_found_counts = c_options ? found_lens : nullptr; + auto total_keys = reduce_n(tasks.lengths, tasks.count, 0ul); + auto keys_output = *c_found_keys = arena.alloc(total_keys, c_error).begin(); + return_on_error(c_error); + // 2. Fetch the data level_iter_uptr_t it; try { it = level_iter_uptr_t(db.NewIterator(options)); @@ -442,24 +439,20 @@ void ukv_scan( // for (ukv_size_t i = 0; i != c_min_tasks_count; ++i) { scan_t task = tasks[i]; it->Seek(to_slice(task.min_key)); + offsets[i] = keys_output - *c_found_keys; ukv_size_t j = 0; for (; it->Valid() && j != task.length; j++, it->Next()) { - std::memcpy(&found_keys[j], it->key().data(), sizeof(ukv_key_t)); - if (c_options) - found_lens[j] = static_cast(it->value().size()); - } - - while (j != task.length) { - found_keys[j] = ukv_key_unknown_k; - if (c_options) - found_lens[j] = ukv_val_len_missing_k; + std::memcpy(&keys_output[j], it->key().data(), sizeof(ukv_key_t)); + *keys_output = static_cast(it->value().size()); + ++keys_output; ++j; } - found_keys += task.length; - found_lens += task.length; + counts[i] = j; } + + offsets[tasks.size()] = keys_output - *c_found_keys; } void ukv_size( // diff --git a/src/backend_rocksdb.cpp b/src/backend_rocksdb.cpp index 3c61f7924..ac86e1d2b 100644 --- a/src/backend_rocksdb.cpp +++ b/src/backend_rocksdb.cpp @@ -488,7 +488,7 @@ void ukv_scan( // ukv_size_t const c_start_keys_stride, ukv_val_len_t const* c_scan_lengths, - ukv_val_len_t const c_scan_lengths_stride, + ukv_size_t const c_scan_lengths_stride, ukv_options_t const c_options, From 44b5492ed61a71d781dd0b18b8cc5f8410ea502b Mon Sep 17 00:00:00 2001 From: violeta Date: Mon, 5 Sep 2022 15:45:31 +0400 Subject: [PATCH 30/30] Refactoring: Implement ukv_scan(...) for backend_rocksdb. unum-cloud#43 --- src/backend_leveldb.cpp | 8 +++---- src/backend_rocksdb.cpp | 48 ++++++++++++++++------------------------- 2 files changed, 23 insertions(+), 33 deletions(-) diff --git a/src/backend_leveldb.cpp b/src/backend_leveldb.cpp index 49734d314..6d1168888 100644 --- a/src/backend_leveldb.cpp +++ b/src/backend_leveldb.cpp @@ -414,9 +414,6 @@ void ukv_scan( // strided_iterator_gt lens {c_scan_lengths, c_scan_lengths_stride}; scans_arg_t tasks {{}, keys, lens, c_min_tasks_count}; - leveldb::ReadOptions options; - options.fill_cache = false; - // 1. Allocate a tape for all the values to be fetched auto offsets = arena.alloc_or_dummy(tasks.count + 1, c_error, c_found_offsets); return_on_error(c_error); @@ -428,6 +425,9 @@ void ukv_scan( // return_on_error(c_error); // 2. Fetch the data + leveldb::ReadOptions options; + options.fill_cache = false; + level_iter_uptr_t it; try { it = level_iter_uptr_t(db.NewIterator(options)); @@ -443,7 +443,7 @@ void ukv_scan( // ukv_size_t j = 0; for (; it->Valid() && j != task.length; j++, it->Next()) { - std::memcpy(&keys_output[j], it->key().data(), sizeof(ukv_key_t)); + std::memcpy(keys_output, it->key().data(), sizeof(ukv_key_t)); *keys_output = static_cast(it->value().size()); ++keys_output; ++j; diff --git a/src/backend_rocksdb.cpp b/src/backend_rocksdb.cpp index ac86e1d2b..ba4b9cc64 100644 --- a/src/backend_rocksdb.cpp +++ b/src/backend_rocksdb.cpp @@ -499,10 +499,9 @@ void ukv_scan( // ukv_arena_t* c_arena, ukv_error_t* c_error) { - if (c_txn && (c_options & ukv_option_read_track_k)) { - *c_error = "RocksDB only supports transparent reads!"; - return; - } + return_if_error(c_db, c_error, uninitialized_state_k, "DataBase is uninitialized"); + + return_if_error(c_txn, c_error, (c_options & ukv_option_read_track_k), "RocksDB only supports transparent reads!"); stl_arena_t arena = prepare_arena(c_arena, {}, c_error); return_on_error(c_error); @@ -514,21 +513,19 @@ void ukv_scan( // strided_iterator_gt lengths {c_scan_lengths, c_scan_lengths_stride}; scans_arg_t tasks {cols, keys, lengths, c_min_tasks_count}; - rocksdb::ReadOptions options; - options.fill_cache = false; - - ukv_size_t total_lengths = reduce_n(tasks.lengths, tasks.count, 0ul); - ukv_size_t total_bytes = total_lengths * sizeof(ukv_key_t); - if (c_options) - total_bytes += total_lengths * sizeof(ukv_val_len_t); + // 1. Allocate a tape for all the values to be fetched + auto offsets = arena.alloc_or_dummy(tasks.count + 1, c_error, c_found_offsets); + return_on_error(c_error); + auto counts = arena.alloc_or_dummy(tasks.count, c_error, c_found_counts); + return_on_error(c_error); - span_gt tape = arena.alloc(total_bytes, c_error); + auto total_keys = reduce_n(tasks.lengths, tasks.count, 0ul); + auto keys_output = *c_found_keys = arena.alloc(total_keys, c_error).begin(); return_on_error(c_error); - ukv_key_t* found_keys = reinterpret_cast(tape.begin()); - ukv_val_len_t* found_lens = reinterpret_cast(found_keys + total_lengths); - *c_found_keys = found_keys; - *c_found_counts = c_options ? found_lens : nullptr; + // 2. Fetch the data + rocksdb::ReadOptions options; + options.fill_cache = false; for (ukv_size_t i = 0; i != c_min_tasks_count; ++i) { scan_t task = tasks[i]; @@ -544,24 +541,17 @@ void ukv_scan( // } ukv_size_t j = 0; - it->Seek(to_slice(task.min_key)); for (; it->Valid() && j != task.length; j++, it->Next()) { - std::memcpy(&found_keys[j], it->key().data(), sizeof(ukv_key_t)); - if (c_options) - found_lens[j] = static_cast(it->value().size()); - } - - while (j != task.length) { - found_keys[j] = ukv_key_unknown_k; - if (c_options) - found_lens[j] = ukv_val_len_missing_k; + std::memcpy(keys_output, it->key().data(), sizeof(ukv_key_t)); + *keys_output = static_cast(it->value().size()); + ++keys_output; ++j; } - found_keys += task.length; - if (c_options) - found_lens += task.length; + counts[i] = j; } + + offsets[tasks.size()] = keys_output - *c_found_keys; } void ukv_size( //