From 8acf394216fdd0a0898d58141a3e94170e3a547c Mon Sep 17 00:00:00 2001 From: Iris Date: Wed, 20 Sep 2023 15:12:59 +0200 Subject: [PATCH] feat: update bot to new indexer --- bot/Cargo.toml | 3 +- bot/src/bot.rs | 154 +++++++++++++++++--------------------- bot/src/config.rs | 13 ++-- bot/src/discord.rs | 3 +- bot/src/indexer_status.rs | 18 +++-- bot/src/main.rs | 21 ++++-- bot/src/models.rs | 14 ++-- bot/src/starknet_utils.rs | 7 ++ bot/src/utils.rs | 23 ++++++ config.template.toml | 11 +-- devnet.config.toml | 29 +++++++ indexer | 2 +- 12 files changed, 175 insertions(+), 123 deletions(-) create mode 100644 bot/src/starknet_utils.rs create mode 100644 bot/src/utils.rs create mode 100644 devnet.config.toml diff --git a/bot/Cargo.toml b/bot/Cargo.toml index 5685504..762aa3c 100644 --- a/bot/Cargo.toml +++ b/bot/Cargo.toml @@ -6,7 +6,6 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -starknet = { git = "https://github.com/Th0rgal/starknet-rs.git", branch = "feat/starknet-id" } mongodb = "2.1.0" bson = "2.0.0" chrono = "0.4" @@ -22,4 +21,6 @@ serde_json = "1.0" lazy_static = "1.4.0" reqwest = "0.11.18" num-integer = "0.1.45" +starknet = "0.6.0" +starknet-id = { git = "https://github.com/starknet-id/starknet-id.rs.git", branch = "master" } diff --git a/bot/src/bot.rs b/bot/src/bot.rs index f4effb0..8082373 100644 --- a/bot/src/bot.rs +++ b/bot/src/bot.rs @@ -6,63 +6,30 @@ use bson::{doc, Bson}; use chrono::{Duration, Utc}; use futures::TryStreamExt; use mongodb::options::FindOneOptions; -use num_integer::Integer; -use starknet::accounts::Account; -use starknet::core::chain_id; +use starknet::core::types::{BlockTag, FunctionCall}; use starknet::{ - accounts::{Call, SingleOwnerAccount}, - core::types::{BlockId, CallFunction, FieldElement}, + accounts::{Account, Call, SingleOwnerAccount}, + core::types::{BlockId, FieldElement}, macros::selector, - providers::{Provider, SequencerGatewayProvider}, + providers::{jsonrpc::HttpTransport, JsonRpcClient, Provider}, signers::LocalWallet, }; -use url::Url; +use starknet_id::encode; use crate::discord::log_msg_and_send_to_discord; use crate::models::{ AggregateResult, AggregateResults, DomainAggregateResult, MetadataDoc, Unzip5, }; +use crate::starknet_utils::create_jsonrpc_client; +use crate::utils::{hex_to_bigdecimal, to_uint256}; use crate::{ config::Config, models::{AppState, Domain}, }; -use bigdecimal::{BigDecimal, FromPrimitive}; +use bigdecimal::BigDecimal; lazy_static::lazy_static! { static ref RENEW_TIME: FieldElement = FieldElement::from_dec_str("365").unwrap(); - static ref TWO_POW_128: BigInt = BigInt::from(2).pow(128); -} - -pub fn get_provider(config: &Config) -> SequencerGatewayProvider { - if config.devnet_provider.is_devnet { - SequencerGatewayProvider::new( - Url::from_str(&config.devnet_provider.gateway).unwrap(), - Url::from_str(&config.devnet_provider.feeder_gateway).unwrap(), - ) - } else if config.devnet_provider.is_testnet { - SequencerGatewayProvider::starknet_alpha_goerli() - } else { - SequencerGatewayProvider::starknet_alpha_mainnet() - } -} - -pub fn get_chainid(config: &Config) -> FieldElement { - if config.devnet_provider.is_devnet || config.devnet_provider.is_testnet { - chain_id::TESTNET - } else { - chain_id::MAINNET - } -} - -fn to_uint256(n: BigInt) -> (FieldElement, FieldElement) { - let (n_high, n_low) = n.div_rem(&TWO_POW_128); - let (_, low_bytes) = n_low.to_bytes_be(); - let (_, high_bytes) = n_high.to_bytes_be(); - - ( - FieldElement::from_byte_slice_be(&low_bytes).unwrap(), - FieldElement::from_byte_slice_be(&high_bytes).unwrap(), - ) } pub async fn get_domains_ready_for_renewal( @@ -74,17 +41,17 @@ pub async fn get_domains_ready_for_renewal( // Define aggregate pipeline let pipeline = vec![ - doc! { "$match": { "_chain.valid_to": Bson::Null } }, - doc! { "$match": { "expiry": { "$lt": Bson::Int32((min_expiry_date.timestamp_millis() / 1000).try_into().unwrap()) } } }, + doc! { "$match": { "_cursor.to": null } }, + doc! { "$match": { "expiry": { "$lt": Bson::Int64(min_expiry_date.timestamp_millis() / 1000) } } }, doc! { "$lookup": { - "from": "auto_renewals", + "from": "auto_renew_flows", "let": { "domain_name": "$domain" }, "pipeline": [ { "$match": { "$expr": { "$and": [ { "$eq": [ "$domain", "$$domain_name" ] }, - { "$eq": [ "$_chain.valid_to", Bson::Null ] } + { "$eq": [ { "$ifNull": [ "$_cursor.to", null ] }, null ] }, ]} } } @@ -93,14 +60,14 @@ pub async fn get_domains_ready_for_renewal( }}, doc! { "$unwind": "$renewal_info" }, doc! { "$lookup": { - "from": "approvals", + "from": "auto_renew_approvals", "let": { "renewer_addr": "$renewal_info.renewer_address" }, "pipeline": [ { "$match": { "$expr": { "$and": [ { "$eq": [ "$renewer", "$$renewer_addr" ] }, - { "$eq": [ "$_chain.valid_to", Bson::Null ] } + { "$eq": [ { "$ifNull": [ "$_cursor.to", null ] }, null ] }, ]} } } @@ -112,12 +79,12 @@ pub async fn get_domains_ready_for_renewal( "domain": 1, "expiry": 1, "renewer_address": "$renewal_info.renewer_address", - "auto_renewal_enabled": "$renewal_info.auto_renewal_enabled", - "approval_value": { "$ifNull": [ "$approval_info.value", "0" ] }, + "auto_renewal_enabled": "$renewal_info.enabled", + "approval_value": { "$ifNull": [ "$approval_info.allowance", "0x0" ] }, "limit_price": "$renewal_info.limit_price", "last_renewal": "$renewal_info.last_renewal", "meta_hash": "$renewal_info.meta_hash", - "_chain": "$renewal_info._chain", + "_cursor": "$renewal_info._cursor", }}, ]; @@ -163,6 +130,7 @@ pub async fn get_domains_ready_for_renewal( ) }) .unzip5(); + Ok(AggregateResults { domains, renewers, @@ -182,60 +150,72 @@ async fn process_aggregate_result( return Ok(None); } - let renewer_addr = FieldElement::from_str(&result.renewer_address)?; - let allowance = BigDecimal::from_str(&result.approval_value)?; - let limit_price = BigDecimal::from_str(&result.limit_price)?; + let renewer_addr = FieldElement::from_hex_be(&result.renewer_address)?; + let allowance = if let Some(approval_value) = result.approval_value { + hex_to_bigdecimal(&approval_value).unwrap() + } else { + BigDecimal::from(0) + }; + // let allowance = hex_to_bigdecimal(&result.approval_value).unwrap(); + let limit_price = hex_to_bigdecimal(&result.limit_price).unwrap(); // get renew price from contract - let provider = get_provider(&config); + let provider = create_jsonrpc_client(&config); let domain_name = result .domain .strip_suffix(".stark") .ok_or_else(|| anyhow::anyhow!("Invalid domain name: {:?}", result.domain))?; - let domain_encoded = starknet::id::encode(domain_name) - .map_err(|_| anyhow!("Failed to encode domain name")) - .context("Error occurred while encoding domain name")?; + let domain_len = domain_name.len(); let call_result = provider - .call_contract( - CallFunction { + .call( + FunctionCall { contract_address: config.contract.pricing, entry_point_selector: selector!("compute_renew_price"), - calldata: vec![domain_encoded, *RENEW_TIME], + calldata: vec![domain_len.into(), *RENEW_TIME], }, - BlockId::Latest, + BlockId::Tag(BlockTag::Latest), ) .await; match call_result { - Ok(res) => { - let renew_price = FieldElement::to_big_decimal(&res.result[1], 18); + Ok(price) => { + let renew_price = FieldElement::to_big_decimal(&price[1], 18); + // Check user has enough allowance & limit_price is not lower than renew_price if renew_price <= allowance && renew_price <= limit_price { - if result.meta_hash != "0" { - let decimal_val = - BigInt::parse_bytes(result.meta_hash.as_str().as_bytes(), 10).unwrap(); - let hex_meta_hash = decimal_val.to_str_radix(16); - let metadata_collection = - state.db_metadata.collection::("metadata"); - if let Some(document) = metadata_collection - .find_one(doc! {"meta_hash": hex_meta_hash}, FindOneOptions::default()) - .await? - { - let tax_state = document.tax_state; - if let Some(state_info) = state.states.states.get(&tax_state) { - let tax_rate = (state_info.rate * 100.0).round() as i32; - let tax_price = - (renew_price * BigDecimal::from(tax_rate)) / BigDecimal::from(100); - return Ok(Some(AggregateResult { - domain: domain_encoded, - renewer_addr, - limit_price, - tax_price, - meta_hash: FieldElement::from_dec_str(&result.meta_hash)?, - })); + // encode domain name + let domain_encoded = encode(domain_name) + .map_err(|_| anyhow!("Failed to encode domain name")) + .context("Error occurred while encoding domain name")?; + if let Some(meta_hash) = result.meta_hash { + if meta_hash != "0" { + let decimal_meta_hash = + BigInt::parse_bytes(meta_hash.trim_start_matches("0x").as_bytes(), 16) + .unwrap(); + let hex_meta_hash = decimal_meta_hash.to_str_radix(16); + let metadata_collection = + state.db_metadata.collection::("metadata"); + if let Some(document) = metadata_collection + .find_one(doc! {"meta_hash": hex_meta_hash}, FindOneOptions::default()) + .await? + { + let tax_state = document.tax_state; + if let Some(state_info) = state.states.states.get(&tax_state) { + let tax_rate = (state_info.rate * 100.0).round() as i32; + let tax_price = (renew_price * BigDecimal::from(tax_rate)) + / BigDecimal::from(100); + return Ok(Some(AggregateResult { + domain: domain_encoded, + renewer_addr, + limit_price, + tax_price, + meta_hash: FieldElement::from_hex_be(&meta_hash)?, + })); + } } } } + Ok(Some(AggregateResult { domain: domain_encoded, renewer_addr, @@ -266,7 +246,7 @@ async fn process_aggregate_result( pub async fn renew_domains( config: &Config, - account: &SingleOwnerAccount, + account: &SingleOwnerAccount, LocalWallet>, mut aggregate_results: AggregateResults, ) -> Result<()> { // If we have more than 400 domains to renew we make multiple transactions to avoid hitting the 2M steps limit @@ -324,7 +304,7 @@ pub async fn renew_domains( pub async fn send_transaction( config: &Config, - account: &SingleOwnerAccount, + account: &SingleOwnerAccount, LocalWallet>, aggregate_results: AggregateResults, ) -> Result<()> { let mut calldata: Vec = Vec::new(); diff --git a/bot/src/config.rs b/bot/src/config.rs index 7dce35b..8f28cdb 100644 --- a/bot/src/config.rs +++ b/bot/src/config.rs @@ -34,13 +34,6 @@ pub_struct!(Clone, Deserialize; Database { connection_string_metadata: String, }); -pub_struct!(Clone, Deserialize; DevnetProvider { - is_devnet: bool, - is_testnet: bool, - gateway: String, - feeder_gateway: String, -}); - pub_struct!(Clone, Deserialize; MyAccount { private_key: FieldElement, address: FieldElement, @@ -60,15 +53,19 @@ pub_struct!(Clone, Deserialize; IndexerServer { server_url: String, }); +pub_struct!(Clone, Deserialize; Rpc { + rpc_url: String, +}); + pub_struct!(Clone, Deserialize; Config { apibara: Apibara, contract: Contract, database: Database, - devnet_provider: DevnetProvider, account: MyAccount, discord: Discord, renewals : Renewals, indexer_server: IndexerServer, + rpc: Rpc, }); pub fn load() -> Config { diff --git a/bot/src/discord.rs b/bot/src/discord.rs index 48f549f..3464c3d 100644 --- a/bot/src/discord.rs +++ b/bot/src/discord.rs @@ -2,6 +2,7 @@ use anyhow::{Context, Result}; use serde_json::json; use serenity::http::Http; use serenity::model::channel::Message; +use starknet_id::decode; use crate::config::Config; use crate::models::AggregateResults; @@ -52,7 +53,7 @@ pub async fn log_domains_renewed( .domains .iter() .zip(aggregate_results.domains.iter()) - .map(|(d, r)| format!("- `{}` by `{}`", &starknet::id::decode(*d), r)) + .map(|(d, r)| format!("- `{}` by `{}`", &decode(*d), r)) .collect::>() .join(" \n") ); diff --git a/bot/src/indexer_status.rs b/bot/src/indexer_status.rs index e44a75f..3dd9455 100644 --- a/bot/src/indexer_status.rs +++ b/bot/src/indexer_status.rs @@ -1,16 +1,19 @@ use anyhow::Result; use starknet::{ - core::types::{BlockId, BlockStatus}, + core::types::{BlockId, BlockStatus, MaybePendingBlockWithTxHashes}, providers::Provider, }; -use crate::{bot::get_provider, config::Config}; +use crate::{config::Config, starknet_utils::create_jsonrpc_client}; pub async fn check_block_status(conf: &Config, block_nb: u64) -> Result { - let provider = get_provider(&conf); + let provider = create_jsonrpc_client(&conf); - match provider.get_block(BlockId::Number(block_nb)).await { - Ok(block) => { + match provider + .get_block_with_tx_hashes(BlockId::Number(block_nb)) + .await + { + Ok(MaybePendingBlockWithTxHashes::Block(block)) => { if block.status == BlockStatus::AcceptedOnL2 || block.status == BlockStatus::Pending { Ok(true) } else { @@ -21,6 +24,7 @@ pub async fn check_block_status(conf: &Config, block_nb: u64) -> Result { Ok(false) } } + Ok(MaybePendingBlockWithTxHashes::PendingBlock(_)) => Ok(true), Err(e) => { println!( "Error while fetching block status {} with error: {}", @@ -31,6 +35,7 @@ pub async fn check_block_status(conf: &Config, block_nb: u64) -> Result { } } +// todo: update pub async fn get_status_from_endpoint(conf: &Config) -> Result> { // Perform the request and get the response as a string let raw_response = reqwest::get(format!( @@ -48,7 +53,8 @@ pub async fn get_status_from_endpoint(conf: &Config) -> Result, - pub valid_from: Option, +pub struct Cursor { + pub to: Option, + pub from: Option, } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -46,11 +46,11 @@ pub struct DomainAggregateResult { pub expiry: Option, pub renewer_address: String, pub auto_renewal_enabled: bool, - pub approval_value: String, + pub approval_value: Option, pub limit_price: String, - pub last_renewal: String, - pub meta_hash: String, - pub _chain: Chain, + pub last_renewal: Option, + pub meta_hash: Option, + pub _cursor: Cursor, } pub struct AggregateResult { diff --git a/bot/src/starknet_utils.rs b/bot/src/starknet_utils.rs new file mode 100644 index 0000000..dc36d66 --- /dev/null +++ b/bot/src/starknet_utils.rs @@ -0,0 +1,7 @@ +use crate::config::Config; +use starknet::providers::{jsonrpc::HttpTransport, JsonRpcClient}; +use url::Url; + +pub fn create_jsonrpc_client(conf: &Config) -> JsonRpcClient { + JsonRpcClient::new(HttpTransport::new(Url::parse(&conf.rpc.rpc_url).unwrap())) +} diff --git a/bot/src/utils.rs b/bot/src/utils.rs new file mode 100644 index 0000000..9591d90 --- /dev/null +++ b/bot/src/utils.rs @@ -0,0 +1,23 @@ +use bigdecimal::{num_bigint::BigInt, BigDecimal}; +use num_integer::Integer; +use starknet::core::types::FieldElement; + +lazy_static::lazy_static! { + static ref TWO_POW_128: BigInt = BigInt::from(2).pow(128); +} + +pub fn to_uint256(n: BigInt) -> (FieldElement, FieldElement) { + let (n_high, n_low) = n.div_rem(&TWO_POW_128); + let (_, low_bytes) = n_low.to_bytes_be(); + let (_, high_bytes) = n_high.to_bytes_be(); + + ( + FieldElement::from_byte_slice_be(&low_bytes).unwrap(), + FieldElement::from_byte_slice_be(&high_bytes).unwrap(), + ) +} + +pub fn hex_to_bigdecimal(hex: &str) -> Option { + let without_prefix = hex.trim_start_matches("0x"); + BigInt::parse_bytes(without_prefix.as_bytes(), 16).map(BigDecimal::from) +} diff --git a/config.template.toml b/config.template.toml index 3423b05..cb56518 100644 --- a/config.template.toml +++ b/config.template.toml @@ -18,12 +18,6 @@ connection_string = "" metadata_name = "goerli" connection_string_metadata = "" -[devnet_provider] -is_devnet = false -is_testnet = true -gateway = "http://localhost:5050/gateway" -feeder_gateway = "http://localhost:5050/feeder_gateway" - [account] private_key = "0x123" address = "0x123" @@ -37,4 +31,7 @@ delay = 86400 # 24 hours [indexer_server] port = 8090 -server_url = "http://0.0.0.0" \ No newline at end of file +server_url = "http://0.0.0.0" + +[rpc] +rpc_url = "https://starknet-goerli.g.alchemy.com/v2/xxxxxxx" \ No newline at end of file diff --git a/devnet.config.toml b/devnet.config.toml new file mode 100644 index 0000000..09bec00 --- /dev/null +++ b/devnet.config.toml @@ -0,0 +1,29 @@ +[apibara] +finality = "Pending" +starting_block = 811445 +batch_size = 1 +stream = "https://goerli.starknet.a5a.ch:443" +stream_dev = "http://localhost:7171" +token = "dna_XAE93oaNu3XDKWbiGfiE" + +[contract] +starknetid = "0x0783a9097b26eae0586373b2ce0ed3529ddc44069d1e0fbc4f66d42b69d6850d" +naming = "0x3bab268e932d2cecd1946f100ae67ce3dff9fd234119ea2f6da57d16d29fce" +pricing = "0x2ad9830baa7d300a50329f258bac290f3895a596c087a9c091093f3d11ee277" +renewal = "0x14acb2b1b5891d7e1c7461e5debd965d5190105662a810b1a43e2830a81c5c5" +erc20 = "0x049d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7" +erc20_testnet = "0x49D36570D4E46F48E99674BD3FCC84644DDD6B96F7C741B1562B82F9E004DC7" + +[database] +name = "auto_renew_indexer" +connection_string = "mongodb+srv://iris:QE7fi3fIIut9rhOk@cluster0.rvetzm1.mongodb.net/?retryWrites=true&w=majority" + +[devnet_provider] +is_devnet = false +is_testnet = true +gateway = "http://localhost:5050/gateway" +feeder_gateway = "http://localhost:5050/feeder_gateway" + +[account] +private_key = "0xe3e70682c2094cac629f6fbed82c07cd" +address = "0x7e00d496e324876bbc8531f2d9a82bf154d1a04a50218ee74cdd372f75a551a" \ No newline at end of file diff --git a/indexer b/indexer index cdcc34e..64623ce 160000 --- a/indexer +++ b/indexer @@ -1 +1 @@ -Subproject commit cdcc34edb7f8122248696bdb82790a80df92f648 +Subproject commit 64623ce10f26e6723581d8c51cd696a753a88df3