From 44b0735597a5dc7573c8b9e65a50789f58a26d7b Mon Sep 17 00:00:00 2001 From: Iris Date: Wed, 20 Sep 2023 15:41:48 +0200 Subject: [PATCH 1/6] feat: update bot to new indexer --- bot/Cargo.toml | 3 +- bot/src/bot.rs | 154 +++++++++++++++++--------------------- bot/src/config.rs | 13 ++-- bot/src/discord.rs | 3 +- bot/src/indexer_status.rs | 18 +++-- bot/src/main.rs | 21 ++++-- bot/src/models.rs | 14 ++-- bot/src/starknet_utils.rs | 7 ++ bot/src/utils.rs | 23 ++++++ config.template.toml | 11 +-- indexer | 2 +- 11 files changed, 146 insertions(+), 123 deletions(-) create mode 100644 bot/src/starknet_utils.rs create mode 100644 bot/src/utils.rs diff --git a/bot/Cargo.toml b/bot/Cargo.toml index 5685504..762aa3c 100644 --- a/bot/Cargo.toml +++ b/bot/Cargo.toml @@ -6,7 +6,6 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -starknet = { git = "https://github.com/Th0rgal/starknet-rs.git", branch = "feat/starknet-id" } mongodb = "2.1.0" bson = "2.0.0" chrono = "0.4" @@ -22,4 +21,6 @@ serde_json = "1.0" lazy_static = "1.4.0" reqwest = "0.11.18" num-integer = "0.1.45" +starknet = "0.6.0" +starknet-id = { git = "https://github.com/starknet-id/starknet-id.rs.git", branch = "master" } diff --git a/bot/src/bot.rs b/bot/src/bot.rs index f4effb0..8082373 100644 --- a/bot/src/bot.rs +++ b/bot/src/bot.rs @@ -6,63 +6,30 @@ use bson::{doc, Bson}; use chrono::{Duration, Utc}; use futures::TryStreamExt; use mongodb::options::FindOneOptions; -use num_integer::Integer; -use starknet::accounts::Account; -use starknet::core::chain_id; +use starknet::core::types::{BlockTag, FunctionCall}; use starknet::{ - accounts::{Call, SingleOwnerAccount}, - core::types::{BlockId, CallFunction, FieldElement}, + accounts::{Account, Call, SingleOwnerAccount}, + core::types::{BlockId, FieldElement}, macros::selector, - providers::{Provider, SequencerGatewayProvider}, + providers::{jsonrpc::HttpTransport, JsonRpcClient, Provider}, signers::LocalWallet, }; -use url::Url; +use starknet_id::encode; use crate::discord::log_msg_and_send_to_discord; use crate::models::{ AggregateResult, AggregateResults, DomainAggregateResult, MetadataDoc, Unzip5, }; +use crate::starknet_utils::create_jsonrpc_client; +use crate::utils::{hex_to_bigdecimal, to_uint256}; use crate::{ config::Config, models::{AppState, Domain}, }; -use bigdecimal::{BigDecimal, FromPrimitive}; +use bigdecimal::BigDecimal; lazy_static::lazy_static! { static ref RENEW_TIME: FieldElement = FieldElement::from_dec_str("365").unwrap(); - static ref TWO_POW_128: BigInt = BigInt::from(2).pow(128); -} - -pub fn get_provider(config: &Config) -> SequencerGatewayProvider { - if config.devnet_provider.is_devnet { - SequencerGatewayProvider::new( - Url::from_str(&config.devnet_provider.gateway).unwrap(), - Url::from_str(&config.devnet_provider.feeder_gateway).unwrap(), - ) - } else if config.devnet_provider.is_testnet { - SequencerGatewayProvider::starknet_alpha_goerli() - } else { - SequencerGatewayProvider::starknet_alpha_mainnet() - } -} - -pub fn get_chainid(config: &Config) -> FieldElement { - if config.devnet_provider.is_devnet || config.devnet_provider.is_testnet { - chain_id::TESTNET - } else { - chain_id::MAINNET - } -} - -fn to_uint256(n: BigInt) -> (FieldElement, FieldElement) { - let (n_high, n_low) = n.div_rem(&TWO_POW_128); - let (_, low_bytes) = n_low.to_bytes_be(); - let (_, high_bytes) = n_high.to_bytes_be(); - - ( - FieldElement::from_byte_slice_be(&low_bytes).unwrap(), - FieldElement::from_byte_slice_be(&high_bytes).unwrap(), - ) } pub async fn get_domains_ready_for_renewal( @@ -74,17 +41,17 @@ pub async fn get_domains_ready_for_renewal( // Define aggregate pipeline let pipeline = vec![ - doc! { "$match": { "_chain.valid_to": Bson::Null } }, - doc! { "$match": { "expiry": { "$lt": Bson::Int32((min_expiry_date.timestamp_millis() / 1000).try_into().unwrap()) } } }, + doc! { "$match": { "_cursor.to": null } }, + doc! { "$match": { "expiry": { "$lt": Bson::Int64(min_expiry_date.timestamp_millis() / 1000) } } }, doc! { "$lookup": { - "from": "auto_renewals", + "from": "auto_renew_flows", "let": { "domain_name": "$domain" }, "pipeline": [ { "$match": { "$expr": { "$and": [ { "$eq": [ "$domain", "$$domain_name" ] }, - { "$eq": [ "$_chain.valid_to", Bson::Null ] } + { "$eq": [ { "$ifNull": [ "$_cursor.to", null ] }, null ] }, ]} } } @@ -93,14 +60,14 @@ pub async fn get_domains_ready_for_renewal( }}, doc! { "$unwind": "$renewal_info" }, doc! { "$lookup": { - "from": "approvals", + "from": "auto_renew_approvals", "let": { "renewer_addr": "$renewal_info.renewer_address" }, "pipeline": [ { "$match": { "$expr": { "$and": [ { "$eq": [ "$renewer", "$$renewer_addr" ] }, - { "$eq": [ "$_chain.valid_to", Bson::Null ] } + { "$eq": [ { "$ifNull": [ "$_cursor.to", null ] }, null ] }, ]} } } @@ -112,12 +79,12 @@ pub async fn get_domains_ready_for_renewal( "domain": 1, "expiry": 1, "renewer_address": "$renewal_info.renewer_address", - "auto_renewal_enabled": "$renewal_info.auto_renewal_enabled", - "approval_value": { "$ifNull": [ "$approval_info.value", "0" ] }, + "auto_renewal_enabled": "$renewal_info.enabled", + "approval_value": { "$ifNull": [ "$approval_info.allowance", "0x0" ] }, "limit_price": "$renewal_info.limit_price", "last_renewal": "$renewal_info.last_renewal", "meta_hash": "$renewal_info.meta_hash", - "_chain": "$renewal_info._chain", + "_cursor": "$renewal_info._cursor", }}, ]; @@ -163,6 +130,7 @@ pub async fn get_domains_ready_for_renewal( ) }) .unzip5(); + Ok(AggregateResults { domains, renewers, @@ -182,60 +150,72 @@ async fn process_aggregate_result( return Ok(None); } - let renewer_addr = FieldElement::from_str(&result.renewer_address)?; - let allowance = BigDecimal::from_str(&result.approval_value)?; - let limit_price = BigDecimal::from_str(&result.limit_price)?; + let renewer_addr = FieldElement::from_hex_be(&result.renewer_address)?; + let allowance = if let Some(approval_value) = result.approval_value { + hex_to_bigdecimal(&approval_value).unwrap() + } else { + BigDecimal::from(0) + }; + // let allowance = hex_to_bigdecimal(&result.approval_value).unwrap(); + let limit_price = hex_to_bigdecimal(&result.limit_price).unwrap(); // get renew price from contract - let provider = get_provider(&config); + let provider = create_jsonrpc_client(&config); let domain_name = result .domain .strip_suffix(".stark") .ok_or_else(|| anyhow::anyhow!("Invalid domain name: {:?}", result.domain))?; - let domain_encoded = starknet::id::encode(domain_name) - .map_err(|_| anyhow!("Failed to encode domain name")) - .context("Error occurred while encoding domain name")?; + let domain_len = domain_name.len(); let call_result = provider - .call_contract( - CallFunction { + .call( + FunctionCall { contract_address: config.contract.pricing, entry_point_selector: selector!("compute_renew_price"), - calldata: vec![domain_encoded, *RENEW_TIME], + calldata: vec![domain_len.into(), *RENEW_TIME], }, - BlockId::Latest, + BlockId::Tag(BlockTag::Latest), ) .await; match call_result { - Ok(res) => { - let renew_price = FieldElement::to_big_decimal(&res.result[1], 18); + Ok(price) => { + let renew_price = FieldElement::to_big_decimal(&price[1], 18); + // Check user has enough allowance & limit_price is not lower than renew_price if renew_price <= allowance && renew_price <= limit_price { - if result.meta_hash != "0" { - let decimal_val = - BigInt::parse_bytes(result.meta_hash.as_str().as_bytes(), 10).unwrap(); - let hex_meta_hash = decimal_val.to_str_radix(16); - let metadata_collection = - state.db_metadata.collection::("metadata"); - if let Some(document) = metadata_collection - .find_one(doc! {"meta_hash": hex_meta_hash}, FindOneOptions::default()) - .await? - { - let tax_state = document.tax_state; - if let Some(state_info) = state.states.states.get(&tax_state) { - let tax_rate = (state_info.rate * 100.0).round() as i32; - let tax_price = - (renew_price * BigDecimal::from(tax_rate)) / BigDecimal::from(100); - return Ok(Some(AggregateResult { - domain: domain_encoded, - renewer_addr, - limit_price, - tax_price, - meta_hash: FieldElement::from_dec_str(&result.meta_hash)?, - })); + // encode domain name + let domain_encoded = encode(domain_name) + .map_err(|_| anyhow!("Failed to encode domain name")) + .context("Error occurred while encoding domain name")?; + if let Some(meta_hash) = result.meta_hash { + if meta_hash != "0" { + let decimal_meta_hash = + BigInt::parse_bytes(meta_hash.trim_start_matches("0x").as_bytes(), 16) + .unwrap(); + let hex_meta_hash = decimal_meta_hash.to_str_radix(16); + let metadata_collection = + state.db_metadata.collection::("metadata"); + if let Some(document) = metadata_collection + .find_one(doc! {"meta_hash": hex_meta_hash}, FindOneOptions::default()) + .await? + { + let tax_state = document.tax_state; + if let Some(state_info) = state.states.states.get(&tax_state) { + let tax_rate = (state_info.rate * 100.0).round() as i32; + let tax_price = (renew_price * BigDecimal::from(tax_rate)) + / BigDecimal::from(100); + return Ok(Some(AggregateResult { + domain: domain_encoded, + renewer_addr, + limit_price, + tax_price, + meta_hash: FieldElement::from_hex_be(&meta_hash)?, + })); + } } } } + Ok(Some(AggregateResult { domain: domain_encoded, renewer_addr, @@ -266,7 +246,7 @@ async fn process_aggregate_result( pub async fn renew_domains( config: &Config, - account: &SingleOwnerAccount, + account: &SingleOwnerAccount, LocalWallet>, mut aggregate_results: AggregateResults, ) -> Result<()> { // If we have more than 400 domains to renew we make multiple transactions to avoid hitting the 2M steps limit @@ -324,7 +304,7 @@ pub async fn renew_domains( pub async fn send_transaction( config: &Config, - account: &SingleOwnerAccount, + account: &SingleOwnerAccount, LocalWallet>, aggregate_results: AggregateResults, ) -> Result<()> { let mut calldata: Vec = Vec::new(); diff --git a/bot/src/config.rs b/bot/src/config.rs index 7dce35b..8f28cdb 100644 --- a/bot/src/config.rs +++ b/bot/src/config.rs @@ -34,13 +34,6 @@ pub_struct!(Clone, Deserialize; Database { connection_string_metadata: String, }); -pub_struct!(Clone, Deserialize; DevnetProvider { - is_devnet: bool, - is_testnet: bool, - gateway: String, - feeder_gateway: String, -}); - pub_struct!(Clone, Deserialize; MyAccount { private_key: FieldElement, address: FieldElement, @@ -60,15 +53,19 @@ pub_struct!(Clone, Deserialize; IndexerServer { server_url: String, }); +pub_struct!(Clone, Deserialize; Rpc { + rpc_url: String, +}); + pub_struct!(Clone, Deserialize; Config { apibara: Apibara, contract: Contract, database: Database, - devnet_provider: DevnetProvider, account: MyAccount, discord: Discord, renewals : Renewals, indexer_server: IndexerServer, + rpc: Rpc, }); pub fn load() -> Config { diff --git a/bot/src/discord.rs b/bot/src/discord.rs index 48f549f..3464c3d 100644 --- a/bot/src/discord.rs +++ b/bot/src/discord.rs @@ -2,6 +2,7 @@ use anyhow::{Context, Result}; use serde_json::json; use serenity::http::Http; use serenity::model::channel::Message; +use starknet_id::decode; use crate::config::Config; use crate::models::AggregateResults; @@ -52,7 +53,7 @@ pub async fn log_domains_renewed( .domains .iter() .zip(aggregate_results.domains.iter()) - .map(|(d, r)| format!("- `{}` by `{}`", &starknet::id::decode(*d), r)) + .map(|(d, r)| format!("- `{}` by `{}`", &decode(*d), r)) .collect::>() .join(" \n") ); diff --git a/bot/src/indexer_status.rs b/bot/src/indexer_status.rs index e44a75f..3dd9455 100644 --- a/bot/src/indexer_status.rs +++ b/bot/src/indexer_status.rs @@ -1,16 +1,19 @@ use anyhow::Result; use starknet::{ - core::types::{BlockId, BlockStatus}, + core::types::{BlockId, BlockStatus, MaybePendingBlockWithTxHashes}, providers::Provider, }; -use crate::{bot::get_provider, config::Config}; +use crate::{config::Config, starknet_utils::create_jsonrpc_client}; pub async fn check_block_status(conf: &Config, block_nb: u64) -> Result { - let provider = get_provider(&conf); + let provider = create_jsonrpc_client(&conf); - match provider.get_block(BlockId::Number(block_nb)).await { - Ok(block) => { + match provider + .get_block_with_tx_hashes(BlockId::Number(block_nb)) + .await + { + Ok(MaybePendingBlockWithTxHashes::Block(block)) => { if block.status == BlockStatus::AcceptedOnL2 || block.status == BlockStatus::Pending { Ok(true) } else { @@ -21,6 +24,7 @@ pub async fn check_block_status(conf: &Config, block_nb: u64) -> Result { Ok(false) } } + Ok(MaybePendingBlockWithTxHashes::PendingBlock(_)) => Ok(true), Err(e) => { println!( "Error while fetching block status {} with error: {}", @@ -31,6 +35,7 @@ pub async fn check_block_status(conf: &Config, block_nb: u64) -> Result { } } +// todo: update pub async fn get_status_from_endpoint(conf: &Config) -> Result> { // Perform the request and get the response as a string let raw_response = reqwest::get(format!( @@ -48,7 +53,8 @@ pub async fn get_status_from_endpoint(conf: &Config) -> Result, - pub valid_from: Option, +pub struct Cursor { + pub to: Option, + pub from: Option, } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -46,11 +46,11 @@ pub struct DomainAggregateResult { pub expiry: Option, pub renewer_address: String, pub auto_renewal_enabled: bool, - pub approval_value: String, + pub approval_value: Option, pub limit_price: String, - pub last_renewal: String, - pub meta_hash: String, - pub _chain: Chain, + pub last_renewal: Option, + pub meta_hash: Option, + pub _cursor: Cursor, } pub struct AggregateResult { diff --git a/bot/src/starknet_utils.rs b/bot/src/starknet_utils.rs new file mode 100644 index 0000000..dc36d66 --- /dev/null +++ b/bot/src/starknet_utils.rs @@ -0,0 +1,7 @@ +use crate::config::Config; +use starknet::providers::{jsonrpc::HttpTransport, JsonRpcClient}; +use url::Url; + +pub fn create_jsonrpc_client(conf: &Config) -> JsonRpcClient { + JsonRpcClient::new(HttpTransport::new(Url::parse(&conf.rpc.rpc_url).unwrap())) +} diff --git a/bot/src/utils.rs b/bot/src/utils.rs new file mode 100644 index 0000000..9591d90 --- /dev/null +++ b/bot/src/utils.rs @@ -0,0 +1,23 @@ +use bigdecimal::{num_bigint::BigInt, BigDecimal}; +use num_integer::Integer; +use starknet::core::types::FieldElement; + +lazy_static::lazy_static! { + static ref TWO_POW_128: BigInt = BigInt::from(2).pow(128); +} + +pub fn to_uint256(n: BigInt) -> (FieldElement, FieldElement) { + let (n_high, n_low) = n.div_rem(&TWO_POW_128); + let (_, low_bytes) = n_low.to_bytes_be(); + let (_, high_bytes) = n_high.to_bytes_be(); + + ( + FieldElement::from_byte_slice_be(&low_bytes).unwrap(), + FieldElement::from_byte_slice_be(&high_bytes).unwrap(), + ) +} + +pub fn hex_to_bigdecimal(hex: &str) -> Option { + let without_prefix = hex.trim_start_matches("0x"); + BigInt::parse_bytes(without_prefix.as_bytes(), 16).map(BigDecimal::from) +} diff --git a/config.template.toml b/config.template.toml index 3423b05..cb56518 100644 --- a/config.template.toml +++ b/config.template.toml @@ -18,12 +18,6 @@ connection_string = "" metadata_name = "goerli" connection_string_metadata = "" -[devnet_provider] -is_devnet = false -is_testnet = true -gateway = "http://localhost:5050/gateway" -feeder_gateway = "http://localhost:5050/feeder_gateway" - [account] private_key = "0x123" address = "0x123" @@ -37,4 +31,7 @@ delay = 86400 # 24 hours [indexer_server] port = 8090 -server_url = "http://0.0.0.0" \ No newline at end of file +server_url = "http://0.0.0.0" + +[rpc] +rpc_url = "https://starknet-goerli.g.alchemy.com/v2/xxxxxxx" \ No newline at end of file diff --git a/indexer b/indexer index cdcc34e..64623ce 160000 --- a/indexer +++ b/indexer @@ -1 +1 @@ -Subproject commit cdcc34edb7f8122248696bdb82790a80df92f648 +Subproject commit 64623ce10f26e6723581d8c51cd696a753a88df3 From 95bfdb5b0567c08b8127384af79f5a0a2c17a27a Mon Sep 17 00:00:00 2001 From: Iris Date: Wed, 20 Sep 2023 16:24:34 +0200 Subject: [PATCH 2/6] feat: add check user balance --- bot/src/bot.rs | 147 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 107 insertions(+), 40 deletions(-) diff --git a/bot/src/bot.rs b/bot/src/bot.rs index 8082373..8f76be6 100644 --- a/bot/src/bot.rs +++ b/bot/src/bot.rs @@ -140,6 +140,36 @@ pub async fn get_domains_ready_for_renewal( }) } +async fn check_user_balance( + config: &Config, + provider: JsonRpcClient, + addr: FieldElement, + limit_price: BigDecimal, +) -> Result> { + let call_balance = provider + .call( + FunctionCall { + contract_address: config.contract.erc20, + entry_point_selector: selector!("balanceOf"), + calldata: vec![addr], + }, + BlockId::Tag(BlockTag::Latest), + ) + .await; + + match call_balance { + Ok(balance) => { + let balance = FieldElement::to_big_decimal(&balance[0], 18); + Ok(Some(balance >= limit_price)) + } + Err(e) => Err(anyhow::anyhow!( + "Error while fetching balance of user {:?} : {:?}", + &addr, + e + )), + } +} + async fn process_aggregate_result( state: &Arc, result: DomainAggregateResult, @@ -181,55 +211,92 @@ async fn process_aggregate_result( match call_result { Ok(price) => { let renew_price = FieldElement::to_big_decimal(&price[1], 18); - // Check user has enough allowance & limit_price is not lower than renew_price - if renew_price <= allowance && renew_price <= limit_price { - // encode domain name - let domain_encoded = encode(domain_name) - .map_err(|_| anyhow!("Failed to encode domain name")) - .context("Error occurred while encoding domain name")?; - if let Some(meta_hash) = result.meta_hash { - if meta_hash != "0" { - let decimal_meta_hash = - BigInt::parse_bytes(meta_hash.trim_start_matches("0x").as_bytes(), 16) - .unwrap(); - let hex_meta_hash = decimal_meta_hash.to_str_radix(16); - let metadata_collection = - state.db_metadata.collection::("metadata"); - if let Some(document) = metadata_collection - .find_one(doc! {"meta_hash": hex_meta_hash}, FindOneOptions::default()) - .await? - { - let tax_state = document.tax_state; - if let Some(state_info) = state.states.states.get(&tax_state) { - let tax_rate = (state_info.rate * 100.0).round() as i32; - let tax_price = (renew_price * BigDecimal::from(tax_rate)) - / BigDecimal::from(100); - return Ok(Some(AggregateResult { - domain: domain_encoded, - renewer_addr, - limit_price, - tax_price, - meta_hash: FieldElement::from_hex_be(&meta_hash)?, - })); + // Check user has enough allowance + if renew_price <= allowance { + // Check limit_price is not lower than renew_price + if renew_price <= limit_price { + // check user balance is greater or equal than limit_price + let has_funds = + check_user_balance(config, provider, renewer_addr, limit_price.clone()) + .await?; + if let Some(false) = has_funds { + log_msg_and_send_to_discord( + &config, + "[Renewal]", + &format!( + "Domain {} cannot be renewed because {} has not enough balance", + result.domain, result.renewer_address + ), + ) + .await; + return Ok(None); + } + + // encode domain name + let domain_encoded = encode(domain_name) + .map_err(|_| anyhow!("Failed to encode domain name")) + .context("Error occurred while encoding domain name")?; + // Check user meta_hash + if let Some(meta_hash) = result.meta_hash { + if meta_hash != "0" { + let decimal_meta_hash = BigInt::parse_bytes( + meta_hash.trim_start_matches("0x").as_bytes(), + 16, + ) + .unwrap(); + let hex_meta_hash = decimal_meta_hash.to_str_radix(16); + let metadata_collection = + state.db_metadata.collection::("metadata"); + if let Some(document) = metadata_collection + .find_one( + doc! {"meta_hash": hex_meta_hash}, + FindOneOptions::default(), + ) + .await? + { + let tax_state = document.tax_state; + if let Some(state_info) = state.states.states.get(&tax_state) { + let tax_rate = (state_info.rate * 100.0).round() as i32; + let tax_price = (renew_price * BigDecimal::from(tax_rate)) + / BigDecimal::from(100); + return Ok(Some(AggregateResult { + domain: domain_encoded, + renewer_addr, + limit_price, + tax_price, + meta_hash: FieldElement::from_hex_be(&meta_hash)?, + })); + } } } } - } - Ok(Some(AggregateResult { - domain: domain_encoded, - renewer_addr, - limit_price, - tax_price: BigDecimal::from(0), - meta_hash: FieldElement::from_str("0")?, - })) + Ok(Some(AggregateResult { + domain: domain_encoded, + renewer_addr, + limit_price, + tax_price: BigDecimal::from(0), + meta_hash: FieldElement::from_str("0")?, + })) + } else { + log_msg_and_send_to_discord( + &config, + "[Renewal]", + &format!( + "Domain {} cannot be renewed because {} has set a limit_price({}) lower than domain price({})", + result.domain, result.renewer_address, limit_price, renew_price + ), + ) + .await; + Ok(None) + } } else { log_msg_and_send_to_discord( &config, "[Renewal]", &format!( - "Domain {} cannot be renewed because {} has not enough allowance", - result.domain, result.renewer_address + "Domain {} cannot be renewed because {} has not enough allowance ({})", + result.domain, result.renewer_address, allowance ), ) .await; From 45635aee2726cf65f2329558ac611cf6656b15be Mon Sep 17 00:00:00 2001 From: Iris Date: Wed, 20 Sep 2023 17:12:38 +0200 Subject: [PATCH 3/6] feat: add call to indexer endpoint --- bot/Cargo.toml | 4 +++ bot/build.rs | 6 +++++ bot/proto/status.proto | 34 +++++++++++++++++++++++++ bot/src/main.rs | 57 +++++++++++++++++++++++------------------- 4 files changed, 75 insertions(+), 26 deletions(-) create mode 100644 bot/build.rs create mode 100644 bot/proto/status.proto diff --git a/bot/Cargo.toml b/bot/Cargo.toml index 7f52d1d..06301ba 100644 --- a/bot/Cargo.toml +++ b/bot/Cargo.toml @@ -25,4 +25,8 @@ starknet = "0.6.0" starknet-id = { git = "https://github.com/starknet-id/starknet-id.rs.git", branch = "master" } serde_derive = "1.0.183" env_logger = "0.10.0" +tonic = "0.10.0" +prost = "0.12.1" +[build-dependencies] +tonic-build = "0.10.0" diff --git a/bot/build.rs b/bot/build.rs new file mode 100644 index 0000000..89858f3 --- /dev/null +++ b/bot/build.rs @@ -0,0 +1,6 @@ +fn main() -> Result<(), Box> { + tonic_build::configure() + .build_server(false) + .compile(&["proto/status.proto"], &["proto"])?; + Ok(()) +} \ No newline at end of file diff --git a/bot/proto/status.proto b/bot/proto/status.proto new file mode 100644 index 0000000..0419b35 --- /dev/null +++ b/bot/proto/status.proto @@ -0,0 +1,34 @@ +// Apibara Sink status server +syntax = "proto3"; + +package apibara.sink.v1; + +service Status { + // Get Sink status. + rpc GetStatus(GetStatusRequest) returns (GetStatusResponse); +} + +// Request for the `GetStatus` method. +message GetStatusRequest {} + +// Response for the `GetStatus` method. +message GetStatusResponse { + // The status of the sink. + SinkStatus status = 1; + // The starting block. + optional uint64 starting_block = 2; + // The current block. + optional uint64 current_block = 3; + // The current head of the chain. + optional uint64 head_block = 4; + // The reason why the sink is not running. + optional string reason = 5; +} + +enum SinkStatus { + SINK_STATUS_UNKNOWN = 0; + // The sink is running. + SINK_STATUS_RUNNING = 1; + // The sink has errored. + SINK_STATUS_ERRORED = 2; +} \ No newline at end of file diff --git a/bot/src/main.rs b/bot/src/main.rs index f95de93..ad39365 100644 --- a/bot/src/main.rs +++ b/bot/src/main.rs @@ -1,5 +1,7 @@ use std::{borrow::Cow, sync::Arc}; +use self::status::status_client::StatusClient; +use self::status::GetStatusRequest; use bot::renew_domains; use bson::doc; use mongodb::{options::ClientOptions, Client as mongoClient}; @@ -9,9 +11,14 @@ use starknet::{ providers::Provider, signers::{LocalWallet, SigningKey}, }; +use starknet_id::decode; use starknet_utils::create_jsonrpc_client; use tokio::time::sleep; +pub mod status { + tonic::include_proto!("apibara.sink.v1"); +} + mod bot; mod config; mod indexer_status; @@ -97,42 +104,40 @@ async fn main() { starknet::accounts::ExecutionEncoding::Legacy, ); + let mut indexer_client = StatusClient::connect("http://localhost:8118") + .await + .unwrap(); + logger.info("Started"); - // todo: passed to false to now, until we have a way to check if the indexer is up to date let mut need_to_check_status = true; + loop { if need_to_check_status { logger.info("Checking indexer status"); - match indexer_status::get_status_from_endpoint(&conf).await { - Ok(block) => { - println!("Block: {}", block); - match indexer_status::check_block_status(&conf, block).await { - Ok(status) => { - if status { - need_to_check_status = false; - logger.info("Indexer is up to date, starting renewals") - } else { - tokio::time::sleep(std::time::Duration::from_secs(5)).await; - continue; - } - } - Err(error) => { - logger.severe(format!( - "Error while checking block status: {}, retrying in 5 seconds", - error - )); - tokio::time::sleep(std::time::Duration::from_secs(5)).await; - } - } + let request = tonic::Request::new(GetStatusRequest {}); + match indexer_client.get_status(request).await { + Ok(response) => { + println!("RESPONSE={:?}", response.into_inner()); + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + // if status { + // need_to_check_status = false; + // logger.info("Indexer is up to date, starting renewals") + // } else { + // tokio::time::sleep(std::time::Duration::from_secs(5)).await; + // continue; + // } } Err(error) => { - println!( - "Error getting indexer status, retrying in 5 seconds: {}", + println!("ERROR={:?}", error); + logger.severe(format!( + "Error while checking block status: {}, retrying in 5 seconds", error - ); + )); tokio::time::sleep(std::time::Duration::from_secs(5)).await; } } + + tokio::time::sleep(std::time::Duration::from_secs(5)).await; } else { println!("[bot] Checking domains to renew"); match bot::get_domains_ready_for_renewal(&conf, &shared_state, &logger).await { @@ -150,7 +155,7 @@ async fn main() { .for_each(|(d, r)| { logger.info(format!( "- `Renewal: {}` by `{:#x}`", - &starknet::id::decode(*d), + &decode(*d), r )) }); From db138fd0d3bb625cbc500d00a68d402a038d30b3 Mon Sep 17 00:00:00 2001 From: Iris Date: Thu, 21 Sep 2023 09:53:46 +0200 Subject: [PATCH 4/6] fix: protoc build error --- bot/Cargo.toml | 1 + bot/build.rs | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bot/Cargo.toml b/bot/Cargo.toml index 06301ba..d39a101 100644 --- a/bot/Cargo.toml +++ b/bot/Cargo.toml @@ -30,3 +30,4 @@ prost = "0.12.1" [build-dependencies] tonic-build = "0.10.0" +prost-build = "0.12.1" diff --git a/bot/build.rs b/bot/build.rs index 89858f3..ba19fb6 100644 --- a/bot/build.rs +++ b/bot/build.rs @@ -1,6 +1,8 @@ fn main() -> Result<(), Box> { + let mut config = prost_build::Config::new(); + config.protoc_arg("--experimental_allow_proto3_optional"); tonic_build::configure() .build_server(false) .compile(&["proto/status.proto"], &["proto"])?; Ok(()) -} \ No newline at end of file +} From 39ffc52c9410cbfd978c17c4702e6d7d8cb42f92 Mon Sep 17 00:00:00 2001 From: Iris Date: Thu, 21 Sep 2023 11:53:53 +0200 Subject: [PATCH 5/6] fix: protoc error --- bot/Cargo.toml | 1 - bot/build.rs | 3 +-- bot/src/main.rs | 9 ++++++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/bot/Cargo.toml b/bot/Cargo.toml index d39a101..06301ba 100644 --- a/bot/Cargo.toml +++ b/bot/Cargo.toml @@ -30,4 +30,3 @@ prost = "0.12.1" [build-dependencies] tonic-build = "0.10.0" -prost-build = "0.12.1" diff --git a/bot/build.rs b/bot/build.rs index ba19fb6..40da1bf 100644 --- a/bot/build.rs +++ b/bot/build.rs @@ -1,8 +1,7 @@ fn main() -> Result<(), Box> { - let mut config = prost_build::Config::new(); - config.protoc_arg("--experimental_allow_proto3_optional"); tonic_build::configure() .build_server(false) + .protoc_arg("--experimental_allow_proto3_optional") .compile(&["proto/status.proto"], &["proto"])?; Ok(()) } diff --git a/bot/src/main.rs b/bot/src/main.rs index ad39365..8d495ca 100644 --- a/bot/src/main.rs +++ b/bot/src/main.rs @@ -104,9 +104,12 @@ async fn main() { starknet::accounts::ExecutionEncoding::Legacy, ); - let mut indexer_client = StatusClient::connect("http://localhost:8118") - .await - .unwrap(); + let mut indexer_client = StatusClient::connect(format!( + "{}:{}", + conf.indexer_server.server_url, conf.indexer_server.port + )) + .await + .unwrap(); logger.info("Started"); let mut need_to_check_status = true; From f193618b834c4cd17e8d4e528074b232436c6bec Mon Sep 17 00:00:00 2001 From: Iris Date: Thu, 21 Sep 2023 14:04:22 +0200 Subject: [PATCH 6/6] ref: update get status from indexer --- bot/src/config.rs | 2 +- bot/src/indexer_status.rs | 61 ------------------------------- bot/src/main.rs | 75 +++++++++++++++++++++++---------------- config.template.toml | 2 +- 4 files changed, 47 insertions(+), 93 deletions(-) delete mode 100644 bot/src/indexer_status.rs diff --git a/bot/src/config.rs b/bot/src/config.rs index 2b31f0c..fba5e39 100644 --- a/bot/src/config.rs +++ b/bot/src/config.rs @@ -44,7 +44,7 @@ pub_struct!(Clone, Deserialize; Renewals { }); pub_struct!(Clone, Deserialize; IndexerServer { - port: u16, + port: Vec, server_url: String, }); diff --git a/bot/src/indexer_status.rs b/bot/src/indexer_status.rs deleted file mode 100644 index 3dd9455..0000000 --- a/bot/src/indexer_status.rs +++ /dev/null @@ -1,61 +0,0 @@ -use anyhow::Result; -use starknet::{ - core::types::{BlockId, BlockStatus, MaybePendingBlockWithTxHashes}, - providers::Provider, -}; - -use crate::{config::Config, starknet_utils::create_jsonrpc_client}; - -pub async fn check_block_status(conf: &Config, block_nb: u64) -> Result { - let provider = create_jsonrpc_client(&conf); - - match provider - .get_block_with_tx_hashes(BlockId::Number(block_nb)) - .await - { - Ok(MaybePendingBlockWithTxHashes::Block(block)) => { - if block.status == BlockStatus::AcceptedOnL2 || block.status == BlockStatus::Pending { - Ok(true) - } else { - println!( - "Indexer is still processing old data. Block = {}.", - block_nb - ); - Ok(false) - } - } - Ok(MaybePendingBlockWithTxHashes::PendingBlock(_)) => Ok(true), - Err(e) => { - println!( - "Error while fetching block status {} with error: {}", - block_nb, e - ); - Ok(false) - } - } -} - -// todo: update -pub async fn get_status_from_endpoint(conf: &Config) -> Result> { - // Perform the request and get the response as a string - let raw_response = reqwest::get(format!( - "{}:{}/is_ready", - conf.indexer_server.server_url, conf.indexer_server.port - )) - .await? - .text() - .await?; - - // Log the raw response - println!("Raw response: {}", &raw_response); - - // Now, try to parse the raw response as JSON - let response: serde_json::Value = serde_json::from_str(&raw_response)?; - - // Extract the "last_block" field as a usize - let last_block = response["last_block"] - .as_u64() - .ok_or("Failed to parse 'last_block' as usize")?; - - Ok(last_block) -} diff --git a/bot/src/main.rs b/bot/src/main.rs index 8d495ca..9b0a5fb 100644 --- a/bot/src/main.rs +++ b/bot/src/main.rs @@ -21,7 +21,6 @@ pub mod status { mod bot; mod config; -mod indexer_status; mod logger; mod models; mod sales_tax; @@ -104,42 +103,58 @@ async fn main() { starknet::accounts::ExecutionEncoding::Legacy, ); - let mut indexer_client = StatusClient::connect(format!( - "{}:{}", - conf.indexer_server.server_url, conf.indexer_server.port - )) - .await - .unwrap(); - logger.info("Started"); let mut need_to_check_status = true; - loop { if need_to_check_status { logger.info("Checking indexer status"); - let request = tonic::Request::new(GetStatusRequest {}); - match indexer_client.get_status(request).await { - Ok(response) => { - println!("RESPONSE={:?}", response.into_inner()); - tokio::time::sleep(std::time::Duration::from_secs(5)).await; - // if status { - // need_to_check_status = false; - // logger.info("Indexer is up to date, starting renewals") - // } else { - // tokio::time::sleep(std::time::Duration::from_secs(5)).await; - // continue; - // } - } - Err(error) => { - println!("ERROR={:?}", error); - logger.severe(format!( - "Error while checking block status: {}, retrying in 5 seconds", - error - )); - tokio::time::sleep(std::time::Duration::from_secs(5)).await; + let mut is_ready = true; + 'outer: for port in &conf.indexer_server.port { + let indexer_client = + StatusClient::connect(format!("{}:{}", conf.indexer_server.server_url, port)) + .await; + match indexer_client { + Ok(mut indexer) => { + let request = tonic::Request::new(GetStatusRequest {}); + match indexer.get_status(request).await { + Ok(response) => { + let res = response.into_inner(); + if let (Some(current_block), Some(head_block)) = + (res.current_block, res.head_block) + { + if current_block < head_block { + logger.info(format!( + "Indexer on port {} is not up to date. Current block {} is lower than head block {}. Retrying in 5 seconds.", + port, current_block, head_block + )); + is_ready = false; + break 'outer; + } + } + } + Err(e) => { + logger.severe(format!( + "Unable to connect to indexer on port {}: {}", + port, e + )); + is_ready = false; + break 'outer; + } + } + } + Err(e) => { + logger.severe(format!( + "Unable to connect to indexer on port {}: {}", + port, e + )); + continue; + } } } - + if is_ready { + need_to_check_status = false; + logger.info("Indexer is up to date, starting renewals"); + } tokio::time::sleep(std::time::Duration::from_secs(5)).await; } else { println!("[bot] Checking domains to renew"); diff --git a/config.template.toml b/config.template.toml index 2f8956c..f4b9cd7 100644 --- a/config.template.toml +++ b/config.template.toml @@ -26,7 +26,7 @@ address = "0x123" delay = 86400 # 24 hours [indexer_server] -port = 8090 +port = [8005, 8007, 8008] server_url = "http://0.0.0.0" [watchtower]