From 14e59bb527ed09c35ac55fb66f672a22f9792bb9 Mon Sep 17 00:00:00 2001 From: renancloudwalk <53792026+renancloudwalk@users.noreply.github.com> Date: Mon, 25 Mar 2024 13:21:06 -0300 Subject: [PATCH] Create missing partitions for logs and transacitons (#437) * chore: create partitions for block number * chore: add partitions for logs and transactions --- ...68c2f47d09b1325cc53035929d60c89559171.json | 16 ---------- ...fa84f3d62c5ab9d0c75f4e2b32a8232b48d33.json | 18 ----------- ...9277992cb0381970dff6a18896f0cb752185e.json | 17 ++++++++++ ...26299c422ac7a6dc3ef3cf56a2ae789b765af.json | 19 ++++++++++++ src/eth/primitives/hash.rs | 5 +++ src/eth/storage/hybrid/query_executor.rs | 22 ++++++++----- static/schema/001-init.sql | 31 ++++++++++++++++--- 7 files changed, 81 insertions(+), 47 deletions(-) delete mode 100644 .sqlx/query-0b15e5777d491cb4a4fea24045f68c2f47d09b1325cc53035929d60c89559171.json delete mode 100644 .sqlx/query-673c1b1108abf013843370ec99afa84f3d62c5ab9d0c75f4e2b32a8232b48d33.json create mode 100644 .sqlx/query-8e19eca16239dc7685d111fcdd69277992cb0381970dff6a18896f0cb752185e.json create mode 100644 .sqlx/query-e0a6fec1d7b673876167656a05a26299c422ac7a6dc3ef3cf56a2ae789b765af.json diff --git a/.sqlx/query-0b15e5777d491cb4a4fea24045f68c2f47d09b1325cc53035929d60c89559171.json b/.sqlx/query-0b15e5777d491cb4a4fea24045f68c2f47d09b1325cc53035929d60c89559171.json deleted file mode 100644 index 7c714bfaa..000000000 --- a/.sqlx/query-0b15e5777d491cb4a4fea24045f68c2f47d09b1325cc53035929d60c89559171.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO public.neo_transactions (block_number, hash, transaction_data)\n SELECT * FROM UNNEST($1::bigint[], $2::bytea[], $3::jsonb[])\n AS t(block_number, hash, transaction_data);", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8Array", - "ByteaArray", - "JsonbArray" - ] - }, - "nullable": [] - }, - "hash": "0b15e5777d491cb4a4fea24045f68c2f47d09b1325cc53035929d60c89559171" -} diff --git a/.sqlx/query-673c1b1108abf013843370ec99afa84f3d62c5ab9d0c75f4e2b32a8232b48d33.json b/.sqlx/query-673c1b1108abf013843370ec99afa84f3d62c5ab9d0c75f4e2b32a8232b48d33.json deleted file mode 100644 index c2f6acd1e..000000000 --- a/.sqlx/query-673c1b1108abf013843370ec99afa84f3d62c5ab9d0c75f4e2b32a8232b48d33.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO public.neo_logs (block_number, hash, address, log_idx, log_data)\n SELECT * FROM UNNEST($1::bigint[], $2::bytea[], $3::bytea[], $4::numeric[], $5::jsonb[])\n AS t(block_number, hash, address, log_idx, log_data);", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8Array", - "ByteaArray", - "ByteaArray", - "NumericArray", - "JsonbArray" - ] - }, - "nullable": [] - }, - "hash": "673c1b1108abf013843370ec99afa84f3d62c5ab9d0c75f4e2b32a8232b48d33" -} diff --git a/.sqlx/query-8e19eca16239dc7685d111fcdd69277992cb0381970dff6a18896f0cb752185e.json b/.sqlx/query-8e19eca16239dc7685d111fcdd69277992cb0381970dff6a18896f0cb752185e.json new file mode 100644 index 000000000..7199e9771 --- /dev/null +++ b/.sqlx/query-8e19eca16239dc7685d111fcdd69277992cb0381970dff6a18896f0cb752185e.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO public.neo_transactions (block_number, hash, transaction_data, hash_partition)\n SELECT * FROM UNNEST($1::bigint[], $2::bytea[], $3::jsonb[], $4::smallint[])\n AS t(block_number, hash, transaction_data, hash_partition);", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8Array", + "ByteaArray", + "JsonbArray", + "Int2Array" + ] + }, + "nullable": [] + }, + "hash": "8e19eca16239dc7685d111fcdd69277992cb0381970dff6a18896f0cb752185e" +} diff --git a/.sqlx/query-e0a6fec1d7b673876167656a05a26299c422ac7a6dc3ef3cf56a2ae789b765af.json b/.sqlx/query-e0a6fec1d7b673876167656a05a26299c422ac7a6dc3ef3cf56a2ae789b765af.json new file mode 100644 index 000000000..b1becea29 --- /dev/null +++ b/.sqlx/query-e0a6fec1d7b673876167656a05a26299c422ac7a6dc3ef3cf56a2ae789b765af.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO public.neo_logs (block_number, hash, address, log_idx, log_data, hash_partition)\n SELECT * FROM UNNEST($1::bigint[], $2::bytea[], $3::bytea[], $4::numeric[], $5::jsonb[], $6::smallint[])\n AS t(block_number, hash, address, log_idx, log_data, hash_partition);", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8Array", + "ByteaArray", + "ByteaArray", + "NumericArray", + "JsonbArray", + "Int2Array" + ] + }, + "nullable": [] + }, + "hash": "e0a6fec1d7b673876167656a05a26299c422ac7a6dc3ef3cf56a2ae789b765af" +} diff --git a/src/eth/primitives/hash.rs b/src/eth/primitives/hash.rs index 7f95b8265..93021b854 100644 --- a/src/eth/primitives/hash.rs +++ b/src/eth/primitives/hash.rs @@ -39,6 +39,11 @@ impl Hash { pub fn zero() -> Self { Self(H256::zero()) } + + pub fn into_u8(self) -> u8 { + let n = self.0.to_low_u64_ne() % 10; + n as u8 + } } impl Display for Hash { diff --git a/src/eth/storage/hybrid/query_executor.rs b/src/eth/storage/hybrid/query_executor.rs index 2bcbe7716..80f043f3a 100644 --- a/src/eth/storage/hybrid/query_executor.rs +++ b/src/eth/storage/hybrid/query_executor.rs @@ -82,19 +82,23 @@ pub async fn commit_eventually(pool: Arc>, block_task: BlockTask) accounts_changes.4.push(nonce); } - let mut transaction_batch = (Vec::new(), Vec::new(), Vec::new()); - let mut logs_batch = (Vec::new(), Vec::new(), Vec::new(), Vec::new(), Vec::new()); + let mut transaction_batch = (Vec::new(), Vec::new(), Vec::new(), Vec::new()); + let mut logs_batch = (Vec::new(), Vec::new(), Vec::new(), Vec::new(), Vec::new(), Vec::new()); for transaction in block_task.block_data.transactions { + let last_char_vec: u8 = transaction.input.hash.clone().into_u8(); + for log in transaction.logs.iter() { logs_batch.0.push(transaction.block_number); logs_batch.1.push(transaction.input.hash.clone()); logs_batch.2.push(log.log.address.clone()); logs_batch.3.push(log.log_index); logs_batch.4.push(serde_json::to_value(log).unwrap()); + logs_batch.5.push(last_char_vec); } transaction_batch.0.push(transaction.block_number); transaction_batch.1.push(transaction.input.hash.clone()); transaction_batch.2.push(serde_json::to_value(transaction).unwrap()); + transaction_batch.3.push(last_char_vec); } let pool_clone = Arc::>::clone(&pool); @@ -144,12 +148,13 @@ pub async fn commit_eventually(pool: Arc>, block_task: BlockTask) if !transaction_batch.0.is_empty() { sqlx::query!( - "INSERT INTO public.neo_transactions (block_number, hash, transaction_data) - SELECT * FROM UNNEST($1::bigint[], $2::bytea[], $3::jsonb[]) - AS t(block_number, hash, transaction_data);", + "INSERT INTO public.neo_transactions (block_number, hash, transaction_data, hash_partition) + SELECT * FROM UNNEST($1::bigint[], $2::bytea[], $3::jsonb[], $4::smallint[]) + AS t(block_number, hash, transaction_data, hash_partition);", transaction_batch.0 as _, transaction_batch.1 as _, transaction_batch.2 as _, + transaction_batch.3 as _, ) .execute(&mut *tx) .await?; @@ -157,14 +162,15 @@ pub async fn commit_eventually(pool: Arc>, block_task: BlockTask) if !logs_batch.0.is_empty() { sqlx::query!( - "INSERT INTO public.neo_logs (block_number, hash, address, log_idx, log_data) - SELECT * FROM UNNEST($1::bigint[], $2::bytea[], $3::bytea[], $4::numeric[], $5::jsonb[]) - AS t(block_number, hash, address, log_idx, log_data);", + "INSERT INTO public.neo_logs (block_number, hash, address, log_idx, log_data, hash_partition) + SELECT * FROM UNNEST($1::bigint[], $2::bytea[], $3::bytea[], $4::numeric[], $5::jsonb[], $6::smallint[]) + AS t(block_number, hash, address, log_idx, log_data, hash_partition);", logs_batch.0 as _, logs_batch.1 as _, logs_batch.2 as _, logs_batch.3 as _, logs_batch.4 as _, + logs_batch.5 as _, ) .execute(&mut *tx) .await?; diff --git a/static/schema/001-init.sql b/static/schema/001-init.sql index c8959a363..a69092d70 100644 --- a/static/schema/001-init.sql +++ b/static/schema/001-init.sql @@ -786,21 +786,42 @@ END$$; CREATE TABLE public.neo_transactions ( hash BYTEA NOT NULL, + hash_partition SMALLINT NOT NULL, block_number BIGINT NOT NULL, transaction_data JSONB NOT NULL, created_at TIMESTAMP WITHOUT TIME ZONE DEFAULT now(), - PRIMARY KEY (hash) -); + PRIMARY KEY (hash, hash_partition) +) PARTITION BY LIST (hash_partition); +CREATE INDEX idx_neo_transactions_hash ON public.neo_transactions (hash); CREATE TABLE public.neo_logs ( hash BYTEA NOT NULL, + hash_partition SMALLINT NOT NULL, block_number BIGINT NOT NULL, - log_idx numeric NOT NULL, + log_idx NUMERIC NOT NULL, address BYTEA NOT NULL, log_data JSONB NOT NULL, created_at TIMESTAMP WITHOUT TIME ZONE DEFAULT now(), - PRIMARY KEY (hash, block_number, log_idx) -); + PRIMARY KEY (hash, block_number, log_idx, hash_partition) +) PARTITION BY LIST (hash_partition); +CREATE INDEX idx_neo_logs_hash ON public.neo_logs (hash); + +DO $$ +DECLARE + partition_id INT; +BEGIN + FOR partition_id IN 0..9 LOOP + -- Create partitions for public.neo_transactions + EXECUTE format('CREATE TABLE public.neo_transactions_%s PARTITION OF public.neo_transactions FOR VALUES IN (%L);', partition_id, partition_id); + EXECUTE format('CREATE INDEX ON public.neo_transactions_%s (hash_partition);', partition_id); + EXECUTE format('CREATE INDEX ON public.neo_transactions_%s (hash);', partition_id); + + -- Create partitions for public.neo_logs + EXECUTE format('CREATE TABLE public.neo_logs_%s PARTITION OF public.neo_logs FOR VALUES IN (%L);', partition_id, partition_id); + EXECUTE format('CREATE INDEX ON public.neo_logs_%s (hash_partition);', partition_id); + EXECUTE format('CREATE INDEX ON public.neo_logs_%s (hash);', partition_id); + END LOOP; +END$$; -- XXX END