use xor filters instead of bloom filters (#145)

subsquid · Feb 21, 2023 · b4d21e3 · b4d21e3
1 parent 6be4333
commit b4d21e3
Show file tree

Hide file tree

Showing 8 changed files with 41 additions and 270 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/core/src/deserialize.rs b/core/src/deserialize.rs
@@ -8,7 +8,7 @@ use std::result::Result as StdResult;
 #[derive(Debug, Clone, derive_more::Deref, derive_more::From, PartialEq, Eq)]
 pub struct Bytes32(pub Box<[u8; 32]>);
 
-#[derive(Debug, Clone, derive_more::Deref, derive_more::From, PartialEq, Eq, Hash)]
+#[derive(Debug, Clone, derive_more::Deref, derive_more::From, PartialEq, Eq)]
 pub struct Address(pub Box<[u8; 20]>);
 
 #[derive(Debug, Clone, derive_more::Deref, derive_more::From, PartialEq, Eq)]

diff --git a/worker/Cargo.toml b/worker/Cargo.toml
@@ -22,9 +22,7 @@ futures = "0.3"
 mimalloc = { version = "0.1.34", default-features = false }
 crossbeam-channel = "0.5"
 derive_more = "0.99"
-fnv = "1"
-bv = { version = "0.11", features = [ "serde" ] }
-rand = "0.8"
+xorf = { version = "0.8.1", features = ["serde"] }
 
 eth-archive-core = { path = "../core" }
 

diff --git a/worker/src/bloom.rs b/worker/src/bloom.rs
diff --git a/worker/src/data_ctx.rs b/worker/src/data_ctx.rs
@@ -1,6 +1,6 @@
 use crate::config::Config;
 use crate::db::DbHandle;
-use crate::db_writer::DbWriter;
+use crate::db_writer::{hash_addr, DbWriter};
 use crate::field_selection::FieldSelection;
 use crate::serialize_task::SerializeTask;
 use crate::types::{MiniLogSelection, MiniQuery, MiniTransactionSelection, Query};
@@ -23,6 +23,7 @@ use std::path::Path;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 use std::{cmp, io};
+use xorf::Filter;
 
 pub struct DataCtx {
     config: Config,
@@ -219,7 +220,7 @@ impl DataCtx {
 
                             let address = address
                                 .iter()
-                                .filter(|addr| parquet_idx.contains(addr))
+                                .filter(|addr| parquet_idx.contains(&hash_addr(addr.as_slice())))
                                 .cloned()
                                 .collect::<Vec<_>>();
 
@@ -242,7 +243,9 @@ impl DataCtx {
                                 Some(source) if !source.is_empty() => {
                                     let source = source
                                         .iter()
-                                        .filter(|addr| parquet_idx.contains(addr))
+                                        .filter(|addr| {
+                                            parquet_idx.contains(&hash_addr(addr.as_slice()))
+                                        })
                                         .cloned()
                                         .collect::<Vec<_>>();
                                     if source.is_empty() {
@@ -258,7 +261,9 @@ impl DataCtx {
                                 Some(dest) if !dest.is_empty() => {
                                     let dest = dest
                                         .iter()
-                                        .filter(|addr| parquet_idx.contains(addr))
+                                        .filter(|addr| {
+                                            parquet_idx.contains(&hash_addr(addr.as_slice()))
+                                        })
                                         .cloned()
                                         .collect::<Vec<_>>();
                                     if dest.is_empty() {

diff --git a/worker/src/db.rs b/worker/src/db.rs
@@ -1,4 +1,3 @@
-use crate::bloom::Bloom as BloomFilter;
 use crate::types::MiniQuery;
 use crate::{Error, Result};
 use eth_archive_core::deserialize::Address;
@@ -15,7 +14,6 @@ use std::sync::Arc;
 use std::time::Instant;
 use std::{cmp, iter, mem};
 
-pub type Bloom = BloomFilter<Address>;
 pub type ParquetIdxIter<'a> = Box<dyn Iterator<Item = Result<(DirName, ParquetIdx)>> + 'a>;
 
 pub struct DbHandle {
@@ -559,7 +557,7 @@ mod cf_name {
     pub const ALL_CF_NAMES: [&str; 5] = [BLOCK, TX, LOG, LOG_TX, PARQUET_IDX];
 }
 
-pub type ParquetIdx = Bloom;
+pub type ParquetIdx = xorf::BinaryFuse16;
 
 fn log_tx_key(block_number: u32, transaction_index: u32) -> [u8; 8] {
     let mut key = [0; 8];