From f283413b34c4802f36047355d6cf87fa16758dfe Mon Sep 17 00:00:00 2001 From: Tiago Carvalho Date: Fri, 9 Aug 2024 09:26:22 +0100 Subject: [PATCH] include height in masp txs block index --- block-index/src/main.rs | 74 +++++++++++++------ .../2024-07-23-171457_block_index/up.sql | 3 +- orm/src/block_index.rs | 1 + orm/src/schema.rs | 1 + swagger.yml | 4 + webserver/src/handler/namada_state.rs | 7 +- webserver/src/repository/namada_state.rs | 27 ++++--- webserver/src/response/namada_state.rs | 1 + webserver/src/service/namada_state.rs | 10 ++- 9 files changed, 92 insertions(+), 36 deletions(-) diff --git a/block-index/src/main.rs b/block-index/src/main.rs index 4f80910..05b23a9 100644 --- a/block-index/src/main.rs +++ b/block-index/src/main.rs @@ -197,32 +197,58 @@ async fn build_new_block_index(app_state: &AppState) -> Result<(), MainError> { let conn = app_state.get_db_connection().await.into_db_error()?; - let block_heights = conn + let (last_height, block_heights_with_txs) = conn .interact(|conn| { - use schema::tx::dsl::*; - - tx.select(block_height) - .distinct() - .load_iter::<_, DbDefaultLoadingMode>(conn) - .context("Failed to query block heights with masp txs")? - .try_fold(Vec::new(), |mut accum, maybe_block_height| { - tracing::debug!("Reading block height entry from db"); - let height: i32 = maybe_block_height.context( - "Failed to get tx block height row data from db", - )?; - tracing::debug!("Read block height entry from db"); - accum.push(u64::try_from(height).context( - "Failed to convert block height from i32 to u64", - )?); - anyhow::Ok(accum) - }) + conn.build_transaction().read_only().run(|conn| { + let last_height = { + use diesel::prelude::OptionalExtension; + use schema::chain_state::dsl::*; + + chain_state + .select(block_height) + .first(conn) + .optional() + .context("Failed to query last block height") + }?; + + let block_heights_with_txs = { + use schema::tx::dsl::*; + + tx.select(block_height) + .distinct() + .load_iter::<_, DbDefaultLoadingMode>(conn) + .context("Failed to query block heights with masp txs")? + .try_fold( + Vec::new(), + |mut accum, maybe_block_height| { + tracing::debug!( + "Reading block height entry from db" + ); + let height: i32 = maybe_block_height.context( + "Failed to get tx block height row data \ + from db", + )?; + tracing::debug!( + "Read block height entry from db" + ); + accum.push(u64::try_from(height).context( + "Failed to convert block height from i32 \ + to u64", + )?); + anyhow::Ok(accum) + }, + ) + }?; + + anyhow::Ok((last_height, block_heights_with_txs)) + }) }) .await .context_db_interact_error() .into_db_error()? .into_db_error()?; - let block_heights_len = block_heights.len(); + let block_heights_len = block_heights_with_txs.len(); tracing::debug!( num_blocks_with_masp_txs = block_heights_len, "Read all block heights with masp transactions from db" @@ -234,7 +260,7 @@ async fn build_new_block_index(app_state: &AppState) -> Result<(), MainError> { transactions" ); - let filter: BinaryFuse16 = block_heights + let filter: BinaryFuse16 = block_heights_with_txs .try_into() .map_err(|err| { anyhow!( @@ -260,19 +286,23 @@ async fn build_new_block_index(app_state: &AppState) -> Result<(), MainError> { tracing::debug!("Storing binary fuse xor filter in db"); - conn.interact(|conn| { + conn.interact(move |conn| { use schema::block_index::dsl::*; let db_filter = BlockIndex { id: 0, serialized_data: serialized_filter, + block_height: last_height.unwrap_or_default(), }; diesel::insert_into(block_index) .values(&db_filter) .on_conflict(id) .do_update() - .set(serialized_data.eq(&db_filter.serialized_data)) + .set(( + block_height.eq(&db_filter.block_height), + serialized_data.eq(&db_filter.serialized_data), + )) .execute(conn) .context("Failed to insert masp txs block index into db")?; diff --git a/orm/migrations/2024-07-23-171457_block_index/up.sql b/orm/migrations/2024-07-23-171457_block_index/up.sql index 42d5bc6..8c99fd0 100644 --- a/orm/migrations/2024-07-23-171457_block_index/up.sql +++ b/orm/migrations/2024-07-23-171457_block_index/up.sql @@ -1,5 +1,6 @@ CREATE TABLE block_index ( id SERIAL PRIMARY KEY, -- NB: serialized with `bincode` - serialized_data bytea NOT NULL + serialized_data bytea NOT NULL, + block_height INT NOT NULL ); diff --git a/orm/src/block_index.rs b/orm/src/block_index.rs index 3e662ba..685a215 100644 --- a/orm/src/block_index.rs +++ b/orm/src/block_index.rs @@ -8,4 +8,5 @@ use crate::schema::block_index; pub struct BlockIndex { pub id: i32, pub serialized_data: Vec, + pub block_height: i32, } diff --git a/orm/src/schema.rs b/orm/src/schema.rs index 06d0ffd..015ae63 100644 --- a/orm/src/schema.rs +++ b/orm/src/schema.rs @@ -4,6 +4,7 @@ diesel::table! { block_index (id) { id -> Int4, serialized_data -> Bytea, + block_height -> Int4, } } diff --git a/swagger.yml b/swagger.yml index fb1fa4c..22f58e1 100644 --- a/swagger.yml +++ b/swagger.yml @@ -199,6 +199,10 @@ components: BlockIndexResponse: type: object properties: + block_height: + type: integer + minimum: 0 + description: The block height of the index. index: description: Compressed (lossy) index of all blocks containing masp txs. type: object diff --git a/webserver/src/handler/namada_state.rs b/webserver/src/handler/namada_state.rs index 4b7d1a7..e48cd97 100644 --- a/webserver/src/handler/namada_state.rs +++ b/webserver/src/handler/namada_state.rs @@ -39,8 +39,11 @@ pub async fn get_block_index( NamadaStateError::Database(err.to_string()) })?; - if let Some(index) = maybe_block_index { - Ok(Json(BlockIndexResponse { index })) + if let Some((height, index)) = maybe_block_index { + Ok(Json(BlockIndexResponse { + block_height: height.0, + index, + })) } else { Err(NamadaStateError::BlockIndexNotFound) } diff --git a/webserver/src/repository/namada_state.rs b/webserver/src/repository/namada_state.rs index 1f13532..0ae5cd9 100644 --- a/webserver/src/repository/namada_state.rs +++ b/webserver/src/repository/namada_state.rs @@ -17,7 +17,9 @@ pub trait NamadaStateRepositoryTrait { async fn get_latest_height(&self) -> anyhow::Result>; - async fn get_block_index(&self) -> anyhow::Result>; + async fn get_block_index( + &self, + ) -> anyhow::Result>; } impl NamadaStateRepositoryTrait for NamadaStateRepository { @@ -46,13 +48,15 @@ impl NamadaStateRepositoryTrait for NamadaStateRepository { Ok(block_height.map(BlockHeight::from)) } - async fn get_block_index(&self) -> anyhow::Result> { + async fn get_block_index( + &self, + ) -> anyhow::Result> { let conn = self.app_state.get_db_connection().await.context( "Failed to retrieve connection from the pool of database \ connections", )?; - let maybe_serialized_data = conn + let maybe_index = conn .interact(move |conn| { use orm::block_index::BlockIndex; use orm::schema::block_index::dsl::block_index; @@ -65,8 +69,12 @@ impl NamadaStateRepositoryTrait for NamadaStateRepository { .context("Failed to get latest block index from db")? .map( |BlockIndex { - serialized_data, .. - }| serialized_data, + block_height, + serialized_data, + .. + }| { + (block_height, serialized_data) + }, ), ) }) @@ -74,12 +82,13 @@ impl NamadaStateRepositoryTrait for NamadaStateRepository { .context_db_interact_error()??; tokio::task::block_in_place(|| { - maybe_serialized_data - .map(|data| { - bincode::deserialize(&data).context( + maybe_index + .map(|(height, data)| { + let filter = bincode::deserialize(&data).context( "Failed to deserialize block index data returned from \ db", - ) + )?; + anyhow::Ok((height, filter)) }) .transpose() }) diff --git a/webserver/src/response/namada_state.rs b/webserver/src/response/namada_state.rs index 907a6d0..8ad6698 100644 --- a/webserver/src/response/namada_state.rs +++ b/webserver/src/response/namada_state.rs @@ -8,5 +8,6 @@ pub struct LatestHeightResponse { #[derive(Clone, Debug, Deserialize, Serialize)] pub struct BlockIndexResponse { + pub block_height: u64, pub index: BinaryFuse16, } diff --git a/webserver/src/service/namada_state.rs b/webserver/src/service/namada_state.rs index 129dc52..3ee00d6 100644 --- a/webserver/src/service/namada_state.rs +++ b/webserver/src/service/namada_state.rs @@ -25,7 +25,13 @@ impl NamadaStateService { pub async fn get_block_index( &self, - ) -> anyhow::Result> { - self.namada_state_repo.get_block_index().await + ) -> anyhow::Result> { + self.namada_state_repo + .get_block_index() + .await + .map(|option| { + option + .map(|(height, filter)| (BlockHeight(height as _), filter)) + }) } }