Redo how WAL/logs are limited by the DB

Adds a patch to the latest rocksdb.

Cargo.toml

@@ -3,6 +3,7 @@ resolver = "2"
 members = [
   # Version patches
   "patches/zstd",
+  "patches/rocksdb",
   "patches/proc-macro-crate",
 
   # std patches
@@ -112,6 +113,8 @@ dockertest = { git = "https://github.com/kayabaNerve/dockertest-rs", branch = "a
 
 # wasmtime pulls in an old version for this
 zstd = { path = "patches/zstd" }
+# Needed for WAL compression
+rocksdb = { path = "patches/rocksdb" }
 # proc-macro-crate 2 binds to an old version of toml for msrv so we patch to 3
 proc-macro-crate = { path = "patches/proc-macro-crate" }
 

common/db/Cargo.toml

@@ -18,7 +18,7 @@ workspace = true
 
 [dependencies]
 parity-db = { version = "0.4", default-features = false, optional = true }
-rocksdb = { version = "0.21", default-features = false, features = ["lz4"], optional = true }
+rocksdb = { version = "0.21", default-features = false, features = ["zstd"], optional = true }
 
 [features]
 parity-db = ["dep:parity-db"]

common/db/src/rocks.rs

@@ -1,6 +1,8 @@
 use std::sync::Arc;
 
-use rocksdb::{DBCompressionType, ThreadMode, SingleThreaded, Options, Transaction, TransactionDB};
+use rocksdb::{
+  DBCompressionType, ThreadMode, SingleThreaded, LogLevel, Options, Transaction, TransactionDB,
+};
 
 use crate::*;
 
@@ -37,13 +39,16 @@ pub type RocksDB = Arc<TransactionDB<SingleThreaded>>;
 pub fn new_rocksdb(path: &str) -> RocksDB {
   let mut options = Options::default();
   options.create_if_missing(true);
-  options.set_compression_type(DBCompressionType::Lz4);
-  options.set_wal_size_limit_mb(128);
-  // 1 GB
-  options.set_max_total_wal_size(1 << 30);
+  options.set_compression_type(DBCompressionType::Zstd);
+
   // 128 MB
-  options.set_max_log_file_size(1 << 27);
-  options.set_recycle_log_file_num(5);
-  options.set_keep_log_file_num(5);
+  options.set_wal_compression_type(DBCompressionType::Zstd);
+  options.set_max_total_wal_size(128 * 1024 * 1024);
+
+  // 1 MB
+  options.set_log_level(LogLevel::Warn);
+  options.set_max_log_file_size(1024 * 1024);
+  options.set_recycle_log_file_num(1);
+
   Arc::new(TransactionDB::open(&options, &Default::default(), path).unwrap())
 }

patches/rocksdb/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "rocksdb"
+version = "0.21.0"
+description = "rocksdb which patches to the latest update"
+license = "MIT"
+repository = "https://github.com/serai-dex/serai/tree/develop/patches/rocksdb"
+authors = ["Luke Parker <[email protected]>"]
+keywords = []
+edition = "2021"
+rust-version = "1.70"
+
+[package.metadata.docs.rs]
+all-features = true
+rustdoc-args = ["--cfg", "docsrs"]
+
+[dependencies]
+rocksdb = { version = "0.22", default-features = false }
+
+[features]
+jemalloc = ["rocksdb/jemalloc"]
+snappy = ["rocksdb/snappy"]
+lz4 = ["rocksdb/lz4"]
+zstd = ["rocksdb/zstd"]
+zlib = ["rocksdb/zlib"]
+bzip2 = ["rocksdb/bzip2"]
+default = ["snappy", "lz4", "zstd", "zlib", "bzip2"]

patches/rocksdb/src/lib.rs

@@ -0,0 +1 @@
+pub use rocksdb::*;