increase block cache for 'Id2NodeDb' and 'RocksDbStore'

crates/core/src/kv/rocksdb_store.rs

@@ -51,7 +51,7 @@ where
         options.set_level_compaction_dynamic_level_bytes(true);
         options.set_bytes_per_sync(1048576);
         let mut block_options = BlockBasedOptions::default();
-        block_options.set_block_size(16 * 1024);
+        block_options.set_block_size(1024 * 1024 * 1024); // 1 GB
         block_options.set_format_version(5);
         block_options.set_cache_index_and_filter_blocks(true);
         block_options.set_pin_l0_filter_and_index_blocks_in_cache(true);

crates/core/src/webgraph/mod.rs

@@ -470,6 +470,7 @@ impl Meta {
 
 struct Id2NodeDb {
     db: rocksdb::DB,
+    _cache: rocksdb::Cache, // needs to be kept alive for as long as the db is alive
 }
 
 impl Id2NodeDb {
@@ -484,22 +485,25 @@ impl Id2NodeDb {
         opts.set_target_file_size_base(512 * 1024 * 1024); // 512 MB
         opts.set_target_file_size_multiplier(10);
 
+        opts.set_compression_type(rocksdb::DBCompressionType::Lz4);
+
         let mut block_opts = rocksdb::BlockBasedOptions::default();
+        let cache = rocksdb::Cache::new_lru_cache(8 * 1024 * 1024 * 1024); // 8 gb
+        opts.set_block_based_table_factory(&block_opts);
 
         // some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)
         opts.set_level_compaction_dynamic_level_bytes(true);
         opts.set_bytes_per_sync(1048576);
+
         block_opts.set_block_size(16 * 1024);
         block_opts.set_format_version(5);
         block_opts.set_cache_index_and_filter_blocks(true);
         block_opts.set_pin_l0_filter_and_index_blocks_in_cache(true);
-
-        opts.set_block_based_table_factory(&block_opts);
-        opts.set_compression_type(rocksdb::DBCompressionType::Lz4);
+        block_opts.set_block_cache(&cache);
 
         let db = rocksdb::DB::open(&opts, path).unwrap();
 
-        Self { db }
+        Self { db, _cache: cache }
     }
 
     fn put(&mut self, id: &NodeID, node: &Node) {

crates/core/src/webgraph/store.rs

@@ -202,6 +202,7 @@ impl PrefixDb {
 
         options.set_level_zero_slowdown_writes_trigger(-1);
         options.set_level_zero_stop_writes_trigger(-1);
+        options.set_compression_type(rocksdb::DBCompressionType::None);
 
         // some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)
         options.set_level_compaction_dynamic_level_bytes(true);
@@ -213,7 +214,6 @@ impl PrefixDb {
         block_options.set_pin_l0_filter_and_index_blocks_in_cache(true);
 
         options.set_block_based_table_factory(&block_options);
-        options.set_compression_type(rocksdb::DBCompressionType::Lz4);
 
         let db = rocksdb::DB::open(&options, path).unwrap();