From f58933d5583ab4ef49e7ebd5488a439c3261666b Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 24 Oct 2023 14:01:57 +0200 Subject: [PATCH 01/10] Merged migrations into one file --- .../migrations/20221026000000_init.sql | 59 ++++++++++++------- .../migrations/20230819000000_paid_date.sql | 1 - .../migrations/20230820000000_replaced.sql | 1 - .../migrations/20231017000000_chain_tx.sql | 5 -- .../migrations/20231019000000_scan.sql | 10 ---- .../migrations/20231023000000_chain.sql | 3 - .../erc20_payment_lib/src/db/model/tx_dao.rs | 2 +- crates/erc20_payment_lib/src/db/ops/tx_ops.rs | 52 ++++++++++++++++ 8 files changed, 90 insertions(+), 43 deletions(-) delete mode 100644 crates/erc20_payment_lib/migrations/20230819000000_paid_date.sql delete mode 100644 crates/erc20_payment_lib/migrations/20230820000000_replaced.sql delete mode 100644 crates/erc20_payment_lib/migrations/20231017000000_chain_tx.sql delete mode 100644 crates/erc20_payment_lib/migrations/20231019000000_scan.sql delete mode 100644 crates/erc20_payment_lib/migrations/20231023000000_chain.sql diff --git a/crates/erc20_payment_lib/migrations/20221026000000_init.sql b/crates/erc20_payment_lib/migrations/20221026000000_init.sql index 7cd6223c..08b3abae 100644 --- a/crates/erc20_payment_lib/migrations/20221026000000_init.sql +++ b/crates/erc20_payment_lib/migrations/20221026000000_init.sql @@ -6,25 +6,26 @@ CREATE TABLE "tx" to_addr TEXT NOT NULL, chain_id INTEGER NOT NULL, gas_limit INTEGER NULL, - max_fee_per_gas TEXT NOT NULL, - priority_fee TEXT NOT NULL, + max_fee_per_gas TEXT NULL, + priority_fee TEXT NULL, val TEXT NOT NULL, nonce INTEGER NULL, processing INTEGER NOT NULL, call_data TEXT NULL, - created_date DATETIME NOT NULL, - first_processed DATETIME NULL, + created_date TEXT NOT NULL, + first_processed TEXT NULL, tx_hash TEXT NULL, signed_raw_data TEXT NULL, - signed_date DATETIME NULL, - broadcast_date DATETIME NULL, + signed_date TEXT NULL, + broadcast_date TEXT NULL, broadcast_count INTEGER NOT NULL, - confirm_date DATETIME NULL, + confirm_date TEXT NULL, block_number INTEGER NULL, chain_status INTEGER NULL, fee_paid TEXT NULL, - error TEXT NULL -); + error TEXT NULL, + orig_tx_id INTEGER NULL +) strict; CREATE INDEX "idx_tx_created_date" ON "tx" (created_date); CREATE INDEX "idx_tx_first_processed" ON "tx" (first_processed); @@ -43,14 +44,15 @@ CREATE TABLE "chain_tx" priority_fee TEXT NULL, val TEXT NOT NULL, nonce INTEGER NOT NULL, - checked_date DATETIME NOT NULL, - blockchain_date DATETIME NOT NULL, + checked_date TEXT NOT NULL, + blockchain_date TEXT NOT NULL, block_number INTEGER NOT NULL, chain_status INTEGER NOT NULL, fee_paid TEXT NULL, - error TEXT NULL -); - + error TEXT NULL, + balance_eth TEXT NULL, + balance_glm TEXT NULL +) strict; CREATE TABLE "token_transfer" ( @@ -61,12 +63,13 @@ CREATE TABLE "token_transfer" chain_id INTEGER NOT NULL, token_addr TEXT NULL, token_amount TEXT NOT NULL, - create_date DATETIME NOT NULL, + create_date TEXT NOT NULL, + paid_date TEXT NULL, tx_id INTEGER NULL, fee_paid TEXT NULL, error TEXT NULL, CONSTRAINT "fk_token_transfer_tx" FOREIGN KEY ("tx_id") REFERENCES "tx" ("id") -); +) strict; CREATE TABLE "transfer_in" ( @@ -78,9 +81,9 @@ CREATE TABLE "transfer_in" token_addr TEXT NULL, token_amount TEXT NOT NULL, tx_hash TEXT NULL, - requested_date DATETIME NOT NULL, - received_date DATETIME NULL -); + requested_date TEXT NOT NULL, + received_date TEXT NULL +) strict; CREATE TABLE "chain_transfer" ( @@ -91,8 +94,10 @@ CREATE TABLE "chain_transfer" token_addr TEXT NULL, token_amount TEXT NOT NULL, chain_tx_id INTEGER NOT NULL, + fee_paid TEXT NOT NULL, + blockchain_date TEXT NOT NULL, CONSTRAINT "fk_chain_transfer_tx" FOREIGN KEY ("chain_tx_id") REFERENCES "chain_tx" ("id") -); +) strict; CREATE TABLE "allowance" ( @@ -104,10 +109,20 @@ CREATE TABLE "allowance" chain_id INTEGER NOT NULL, tx_id INTEGER NULL, fee_paid TEXT NULL, - confirm_date DATETIME NULL, + confirm_date TEXT NULL, error TEXT NULL, CONSTRAINT "fk_allowance_tx" FOREIGN KEY ("tx_id") REFERENCES "tx" ("id") -); +) strict; + +CREATE TABLE "scan_info" +( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + chain_id INTEGER NOT NULL, + filter TEXT NOT NULL, + start_block INTEGER NOT NULL, + last_block INTEGER NOT NULL +) strict; +CREATE UNIQUE INDEX "idx_scan_info_chain_id" ON "scan_info" ("chain_id", "filter"); diff --git a/crates/erc20_payment_lib/migrations/20230819000000_paid_date.sql b/crates/erc20_payment_lib/migrations/20230819000000_paid_date.sql deleted file mode 100644 index d58f44b6..00000000 --- a/crates/erc20_payment_lib/migrations/20230819000000_paid_date.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE "token_transfer" ADD COLUMN "paid_date" DATETIME NULL; \ No newline at end of file diff --git a/crates/erc20_payment_lib/migrations/20230820000000_replaced.sql b/crates/erc20_payment_lib/migrations/20230820000000_replaced.sql deleted file mode 100644 index 81904995..00000000 --- a/crates/erc20_payment_lib/migrations/20230820000000_replaced.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE "tx" ADD COLUMN "orig_tx_id" INT NULL; diff --git a/crates/erc20_payment_lib/migrations/20231017000000_chain_tx.sql b/crates/erc20_payment_lib/migrations/20231017000000_chain_tx.sql deleted file mode 100644 index aeb02392..00000000 --- a/crates/erc20_payment_lib/migrations/20231017000000_chain_tx.sql +++ /dev/null @@ -1,5 +0,0 @@ -CREATE UNIQUE INDEX "chain_tx_tx_hash" ON "chain_tx" (tx_hash); - -ALTER TABLE "chain_transfer" ADD COLUMN "fee_paid" TEXT NULL; -ALTER TABLE "chain_transfer" ADD COLUMN "blockchain_date" DATETIME NULL; - diff --git a/crates/erc20_payment_lib/migrations/20231019000000_scan.sql b/crates/erc20_payment_lib/migrations/20231019000000_scan.sql deleted file mode 100644 index 24d978b1..00000000 --- a/crates/erc20_payment_lib/migrations/20231019000000_scan.sql +++ /dev/null @@ -1,10 +0,0 @@ -CREATE TABLE "scan_info" -( - id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - chain_id INTEGER NOT NULL, - filter TEXT NOT NULL, - start_block INTEGER NOT NULL, - last_block INTEGER NOT NULL -); - -CREATE UNIQUE INDEX "idx_scan_info_chain_id" ON "scan_info" ("chain_id", "filter"); \ No newline at end of file diff --git a/crates/erc20_payment_lib/migrations/20231023000000_chain.sql b/crates/erc20_payment_lib/migrations/20231023000000_chain.sql deleted file mode 100644 index 22e44f19..00000000 --- a/crates/erc20_payment_lib/migrations/20231023000000_chain.sql +++ /dev/null @@ -1,3 +0,0 @@ -ALTER TABLE "chain_tx" ADD COLUMN "balance_eth" TEXT NULL; -ALTER TABLE "chain_tx" ADD COLUMN "balance_glm" TEXT NULL; - diff --git a/crates/erc20_payment_lib/src/db/model/tx_dao.rs b/crates/erc20_payment_lib/src/db/model/tx_dao.rs index e682ba72..f19be0bb 100644 --- a/crates/erc20_payment_lib/src/db/model/tx_dao.rs +++ b/crates/erc20_payment_lib/src/db/model/tx_dao.rs @@ -1,7 +1,7 @@ use chrono::{DateTime, Utc}; use serde::Serialize; -#[derive(Serialize, sqlx::FromRow, Debug, Clone)] +#[derive(Serialize, sqlx::FromRow, Debug, Clone, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct TxDao { pub id: i64, diff --git a/crates/erc20_payment_lib/src/db/ops/tx_ops.rs b/crates/erc20_payment_lib/src/db/ops/tx_ops.rs index 6d0e63a9..43fec0aa 100644 --- a/crates/erc20_payment_lib/src/db/ops/tx_ops.rs +++ b/crates/erc20_payment_lib/src/db/ops/tx_ops.rs @@ -239,3 +239,55 @@ WHERE id = $1 .await?; Ok(tx.clone()) } + +#[tokio::test] +async fn tx_test() -> sqlx::Result<()> { + println!("Start tx_test..."); + + use crate::db::create_sqlite_connection; + let conn = create_sqlite_connection(None, None, false, true) + .await + .unwrap(); + + println!("In memory DB created"); + + let mut tx_to_insert = TxDao { + id: -1, + tx_hash: Some("0x13d8a54dec1c0a30f1cd5129f690c3e27b9aadd59504957bad4d247966dadae7".to_string()), + signed_raw_data: None, + signed_date: Some(chrono::Utc::now()), + broadcast_date: Some(chrono::Utc::now()), + broadcast_count: 0, + method: "".to_string(), + from_addr: "0x001066290077e38f222cc6009c0c7a91d5192303".to_string(), + to_addr: "0xbcfe9736a4f5bf2e43620061ff3001ea0d003c0f".to_string(), + chain_id: 987789, + gas_limit: Some(100000), + max_fee_per_gas: "110000000000".to_string(), + priority_fee: "5110000000000".to_string(), + val: "0".to_string(), + nonce: Some(1), + processing: 0, + call_data: None, + created_date: chrono::Utc::now(), + block_number: Some(119677), + chain_status: Some(1), + fee_paid: Some("83779300533141".to_string()), + error: Some("Test error message".to_string()), + orig_tx_id: None, + engine_message: None, + engine_error: None, + first_processed: None, + confirm_date: None + }; + + let tx_from_insert = insert_tx(&conn, &tx_to_insert).await?; + tx_to_insert.id = tx_from_insert.id; + let tx_from_dao = get_transaction(&conn, tx_from_insert.id).await?; + + //all three should be equal + assert_eq!(tx_to_insert, tx_from_dao); + assert_eq!(tx_from_insert, tx_from_dao); + + Ok(()) +} From 7bceff2ec4c2af191fbe25af0e45c0bcd0c23c32 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 24 Oct 2023 14:37:15 +0200 Subject: [PATCH 02/10] Add option for tx gas dao --- .../erc20_payment_lib/src/db/model/tx_dao.rs | 4 +- crates/erc20_payment_lib/src/db/ops/tx_ops.rs | 10 +-- .../erc20_payment_lib/src/sender/allowance.rs | 5 -- .../erc20_payment_lib/src/sender/batching.rs | 16 +---- .../erc20_payment_lib/src/sender/process.rs | 57 +++++++++++---- crates/erc20_payment_lib/src/transaction.rs | 71 ++++++++++++------- 6 files changed, 96 insertions(+), 67 deletions(-) diff --git a/crates/erc20_payment_lib/src/db/model/tx_dao.rs b/crates/erc20_payment_lib/src/db/model/tx_dao.rs index f19be0bb..44099a8c 100644 --- a/crates/erc20_payment_lib/src/db/model/tx_dao.rs +++ b/crates/erc20_payment_lib/src/db/model/tx_dao.rs @@ -10,8 +10,8 @@ pub struct TxDao { pub to_addr: String, pub chain_id: i64, pub gas_limit: Option, - pub max_fee_per_gas: String, - pub priority_fee: String, + pub max_fee_per_gas: Option, + pub priority_fee: Option, pub val: String, pub nonce: Option, pub processing: i64, diff --git a/crates/erc20_payment_lib/src/db/ops/tx_ops.rs b/crates/erc20_payment_lib/src/db/ops/tx_ops.rs index 43fec0aa..dda87bc9 100644 --- a/crates/erc20_payment_lib/src/db/ops/tx_ops.rs +++ b/crates/erc20_payment_lib/src/db/ops/tx_ops.rs @@ -253,7 +253,9 @@ async fn tx_test() -> sqlx::Result<()> { let mut tx_to_insert = TxDao { id: -1, - tx_hash: Some("0x13d8a54dec1c0a30f1cd5129f690c3e27b9aadd59504957bad4d247966dadae7".to_string()), + tx_hash: Some( + "0x13d8a54dec1c0a30f1cd5129f690c3e27b9aadd59504957bad4d247966dadae7".to_string(), + ), signed_raw_data: None, signed_date: Some(chrono::Utc::now()), broadcast_date: Some(chrono::Utc::now()), @@ -263,8 +265,8 @@ async fn tx_test() -> sqlx::Result<()> { to_addr: "0xbcfe9736a4f5bf2e43620061ff3001ea0d003c0f".to_string(), chain_id: 987789, gas_limit: Some(100000), - max_fee_per_gas: "110000000000".to_string(), - priority_fee: "5110000000000".to_string(), + max_fee_per_gas: Some("110000000000".to_string()), + priority_fee: Some("5110000000000".to_string()), val: "0".to_string(), nonce: Some(1), processing: 0, @@ -278,7 +280,7 @@ async fn tx_test() -> sqlx::Result<()> { engine_message: None, engine_error: None, first_processed: None, - confirm_date: None + confirm_date: None, }; let tx_from_insert = insert_tx(&conn, &tx_to_insert).await?; diff --git a/crates/erc20_payment_lib/src/sender/allowance.rs b/crates/erc20_payment_lib/src/sender/allowance.rs index 9c80b0bb..a3363b0c 100644 --- a/crates/erc20_payment_lib/src/sender/allowance.rs +++ b/crates/erc20_payment_lib/src/sender/allowance.rs @@ -22,10 +22,7 @@ pub async fn process_allowance( signer: &impl Signer, ) -> Result { let minimum_allowance: U256 = U256::max_value() / U256::from(2); - let chain_setup = payment_setup.get_chain_setup(allowance_request.chain_id)?; let web3 = payment_setup.get_provider(allowance_request.chain_id)?; - let max_fee_per_gas = chain_setup.max_fee_per_gas; - let priority_fee = chain_setup.priority_fee; let mut db_allowance = find_allowance( conn, @@ -129,8 +126,6 @@ pub async fn process_allowance( Address::from_str(&allowance_request.spender_addr).map_err(err_from!())?, allowance_request.chain_id as u64, None, - max_fee_per_gas, - priority_fee, )?; let mut db_transaction = conn.begin().await.map_err(err_from!())?; let web3_tx_dao = insert_tx(&mut *db_transaction, &approve_tx) diff --git a/crates/erc20_payment_lib/src/sender/batching.rs b/crates/erc20_payment_lib/src/sender/batching.rs index 709c46f7..68383603 100644 --- a/crates/erc20_payment_lib/src/sender/batching.rs +++ b/crates/erc20_payment_lib/src/sender/batching.rs @@ -115,9 +115,6 @@ pub async fn gather_transactions_batch_multi( let use_direct_method = payment_setup.contract_use_direct_method; let use_unpacked_method = payment_setup.contract_use_unpacked_method; - let max_fee_per_gas = chain_setup.max_fee_per_gas; - let priority_fee = chain_setup.priority_fee; - let max_per_batch = chain_setup.multi_contract_max_at_once; log::debug!("Processing token transfer {:?}", token_transfer); if let Some(token_addr) = token_transfer.token_addr.as_ref() { @@ -212,8 +209,6 @@ pub async fn gather_transactions_batch_multi( erc20_amounts[0], token_transfer.chain_id as u64, None, - max_fee_per_gas, - priority_fee, )? } else if let Some(multi_contract_address) = chain_setup.multi_contract_address { log::info!("Inserting transaction stub for ERC20 multi transfer contract: {:?} for {} distinct transfers", multi_contract_address, erc20_to.len()); @@ -225,8 +220,6 @@ pub async fn gather_transactions_batch_multi( erc20_amounts, token_transfer.chain_id as u64, None, - max_fee_per_gas, - priority_fee, use_direct_method, use_unpacked_method, )? @@ -272,7 +265,7 @@ pub async fn gather_transactions_batch( sum += U256::from_dec_str(&token_transfer.token_amount).map_err(err_from!())?; } - let Ok(chain_setup) = payment_setup.get_chain_setup(token_transfer.chain_id) else { + let Ok(_chain_setup) = payment_setup.get_chain_setup(token_transfer.chain_id) else { send_driver_event( &event_sender, DriverEventContent::TransactionFailed(TransactionFailedReason::InvalidChainId( @@ -286,9 +279,6 @@ pub async fn gather_transactions_batch( )); }; - let max_fee_per_gas = chain_setup.max_fee_per_gas; - let priority_fee = chain_setup.priority_fee; - log::debug!("Processing token transfer {:?}", token_transfer); let web3tx = if let Some(token_addr) = token_transfer.token_addr.as_ref() { create_erc20_transfer( @@ -298,8 +288,6 @@ pub async fn gather_transactions_batch( sum, token_transfer.chain_id as u64, None, - max_fee_per_gas, - priority_fee, )? } else { create_eth_transfer( @@ -307,8 +295,6 @@ pub async fn gather_transactions_batch( Address::from_str(&token_transfer.receiver_addr).map_err(err_from!())?, token_transfer.chain_id as u64, None, - max_fee_per_gas, - priority_fee, sum, ) }; diff --git a/crates/erc20_payment_lib/src/sender/process.rs b/crates/erc20_payment_lib/src/sender/process.rs index 0a5a26e8..a7517f31 100644 --- a/crates/erc20_payment_lib/src/sender/process.rs +++ b/crates/erc20_payment_lib/src/sender/process.rs @@ -143,8 +143,16 @@ pub async fn process_transaction( } } } - let max_fee_per_gas = - U256::from_dec_str(&web3_tx_dao.max_fee_per_gas).map_err(err_from!())?; + let mut max_fee_per_gas = if let Some(max_fee_per_gas) = &web3_tx_dao.max_fee_per_gas { + U256::from_dec_str(max_fee_per_gas).map_err(err_from!())? + } else { + chain_setup.max_fee_per_gas + }; + let max_priority_fee = if let Some(priority_fee) = &web3_tx_dao.priority_fee { + U256::from_dec_str(priority_fee).map_err(err_from!())? + } else { + chain_setup.priority_fee + }; if is_polygon_eco_mode { let blockchain_gas_price = web3 @@ -191,8 +199,7 @@ pub async fn process_transaction( } new_target_gas_u256 }; - let tx_priority_fee_u256 = - U256::from_dec_str(&web3_tx_dao.priority_fee).map_err(err_from!())?; + let tx_priority_fee_u256 = max_priority_fee; //max_fee_per_gas cannot be lower than priority fee if new_target_gas_u256 < tx_priority_fee_u256 { @@ -205,10 +212,11 @@ pub async fn process_transaction( new_target_gas_u256.to_gwei().map_err(err_from!())?, ); - web3_tx_dao.max_fee_per_gas = new_target_gas_u256.to_string(); + max_fee_per_gas = new_target_gas_u256; } } - + web3_tx_dao.max_fee_per_gas = Some(max_fee_per_gas.to_string()); + web3_tx_dao.priority_fee = Some(max_priority_fee.to_string()); web3_tx_dao.nonce = Some(nonce); nonce }; @@ -557,9 +565,23 @@ pub async fn process_transaction( .await .map_err(err_from!())?; - let tx_fee_per_gas = web3_tx_dao.max_fee_per_gas.to_gwei().map_err(err_from!())?; + let max_tx_fee_per_gas_str = + web3_tx_dao + .max_fee_per_gas + .clone() + .ok_or(err_create!(TransactionFailedError::new( + "Max fee per gas not found" + )))?; + let max_tx_priority_fee_str = + web3_tx_dao + .priority_fee + .clone() + .ok_or(err_create!(TransactionFailedError::new( + "Priority fee not found" + )))?; + let tx_fee_per_gas = max_tx_fee_per_gas_str.to_gwei().map_err(err_from!())?; let max_fee_per_gas = chain_setup.max_fee_per_gas.to_gwei().map_err(err_from!())?; - let tx_pr_fee_u256 = web3_tx_dao.priority_fee.to_u256().map_err(err_from!())?; + let tx_pr_fee_u256 = max_tx_priority_fee_str.to_u256().map_err(err_from!())?; let tx_pr_fee = tx_pr_fee_u256.to_gwei().map_err(err_from!())?; let config_priority_fee = chain_setup.priority_fee.to_gwei().map_err(err_from!())?; @@ -594,14 +616,14 @@ pub async fn process_transaction( fee_per_gas_bumped_10 = true; log::warn!( "Transaction max fee bumped more than 10% from {} to {} for tx: {}", - web3_tx_dao.max_fee_per_gas, + max_tx_fee_per_gas_str, chain_setup.max_fee_per_gas, web3_tx_dao.id ); } else { log::warn!( "Transaction max fee changed less than 10% more from {} to {} for tx: {}", - web3_tx_dao.max_fee_per_gas, + max_tx_fee_per_gas_str, chain_setup.max_fee_per_gas, web3_tx_dao.id ); @@ -616,12 +638,12 @@ pub async fn process_transaction( priority_fee_changed_10 = true; log::warn!( "Transaction priority fee bumped more than 10% from {} to {} for tx: {}", - web3_tx_dao.priority_fee, + max_tx_priority_fee_str, chain_setup.priority_fee, web3_tx_dao.id ); } else { - log::warn!("Transaction priority fee changed less than 10% more from {} to {} for tx: {}", web3_tx_dao.priority_fee, chain_setup.priority_fee, web3_tx_dao.id); + log::warn!("Transaction priority fee changed less than 10% more from {} to {} for tx: {}", max_tx_priority_fee_str, chain_setup.priority_fee, web3_tx_dao.id); } } @@ -659,8 +681,8 @@ pub async fn process_transaction( to_addr: tx.to_addr.clone(), chain_id: tx.chain_id, gas_limit: tx.gas_limit, - max_fee_per_gas: replacement_max_fee_per_gas.to_string(), - priority_fee: replacement_priority_fee.to_string(), + max_fee_per_gas: Some(replacement_max_fee_per_gas.to_string()), + priority_fee: Some(replacement_priority_fee.to_string()), val: tx.val.clone(), nonce: tx.nonce, processing: tx.processing, @@ -754,8 +776,13 @@ pub async fn process_transaction( .to_gwei() .map_err(err_from!())?; + let max_tx_fee_per_gas_str = + web3_tx_dao.max_fee_per_gas.clone().ok_or(err_create!( + TransactionFailedError::new("Max fee per gas not found") + ))?; + let tx_max_fee_per_gas_gwei = - web3_tx_dao.max_fee_per_gas.to_gwei().map_err(err_from!())?; + max_tx_fee_per_gas_str.to_gwei().map_err(err_from!())?; let assumed_min_priority_fee_gwei = if web3_tx_dao.chain_id == 137 { const POLYGON_MIN_PRIORITY_FEE_FOR_GAS_PRICE_CHECK: u32 = 30; Decimal::from(POLYGON_MIN_PRIORITY_FEE_FOR_GAS_PRICE_CHECK) diff --git a/crates/erc20_payment_lib/src/transaction.rs b/crates/erc20_payment_lib/src/transaction.rs index 019733d0..5c40bf6f 100644 --- a/crates/erc20_payment_lib/src/transaction.rs +++ b/crates/erc20_payment_lib/src/transaction.rs @@ -41,10 +41,22 @@ pub fn dao_to_call_request(web3_tx_dao: &TxDao) -> Result Result, - max_fee_per_gas: U256, - priority_fee: U256, amount: U256, ) -> TxDao { TxDao { @@ -119,8 +141,8 @@ pub fn create_eth_transfer( to_addr: format!("{to:#x}"), chain_id: chain_id as i64, gas_limit: gas_limit.map(|gas_limit| gas_limit as i64), - max_fee_per_gas: max_fee_per_gas.to_string(), - priority_fee: priority_fee.to_string(), + max_fee_per_gas: None, + priority_fee: None, val: amount.to_string(), nonce: None, processing: 1, @@ -149,8 +171,6 @@ pub fn create_eth_transfer_str( to_addr: String, chain_id: u64, gas_limit: Option, - max_fee_per_gas: String, - priority_fee: String, amount: String, ) -> TxDao { TxDao { @@ -160,8 +180,8 @@ pub fn create_eth_transfer_str( to_addr, chain_id: chain_id as i64, gas_limit: gas_limit.map(|gas_limit| gas_limit as i64), - max_fee_per_gas, - priority_fee, + max_fee_per_gas: None, + priority_fee: None, val: amount, nonce: None, processing: 1, @@ -192,8 +212,6 @@ pub fn create_erc20_transfer( erc20_amount: U256, chain_id: u64, gas_limit: Option, - max_fee_per_gas: U256, - priority_fee: U256, ) -> Result { Ok(TxDao { id: 0, @@ -202,8 +220,8 @@ pub fn create_erc20_transfer( to_addr: format!("{token:#x}"), chain_id: chain_id as i64, gas_limit: gas_limit.map(|gas_limit| gas_limit as i64), - max_fee_per_gas: max_fee_per_gas.to_string(), - priority_fee: priority_fee.to_string(), + max_fee_per_gas: None, + priority_fee: None, val: "0".to_string(), nonce: None, processing: 1, @@ -237,8 +255,6 @@ pub fn create_erc20_transfer_multi( erc20_amount: Vec, chain_id: u64, gas_limit: Option, - max_fee_per_gas: U256, - priority_fee: U256, direct: bool, unpacked: bool, ) -> Result { @@ -277,8 +293,8 @@ pub fn create_erc20_transfer_multi( to_addr: format!("{contract:#x}"), chain_id: chain_id as i64, gas_limit: gas_limit.map(|gas_limit| gas_limit as i64), - max_fee_per_gas: max_fee_per_gas.to_string(), - priority_fee: priority_fee.to_string(), + max_fee_per_gas: None, + priority_fee: None, val: "0".to_string(), nonce: None, processing: 1, @@ -307,8 +323,6 @@ pub fn create_erc20_approve( contract_to_approve: Address, chain_id: u64, gas_limit: Option, - max_fee_per_gas: U256, - priority_fee: U256, ) -> Result { Ok(TxDao { id: 0, @@ -317,8 +331,8 @@ pub fn create_erc20_approve( to_addr: format!("{token:#x}"), chain_id: chain_id as i64, gas_limit: gas_limit.map(|gas_limit| gas_limit as i64), - max_fee_per_gas: max_fee_per_gas.to_string(), - priority_fee: priority_fee.to_string(), + max_fee_per_gas: None, + priority_fee: None, val: "0".to_string(), nonce: None, processing: 1, @@ -392,8 +406,13 @@ pub async fn check_transaction( log::info!("Set gas limit basing on gas estimation: {gas_limit}"); web3_tx_dao.gas_limit = Some(gas_limit.as_u64() as i64); - let max_fee_per_gas = - U256::from_dec_str(&web3_tx_dao.max_fee_per_gas).map_err(err_from!())?; + let max_fee_per_gas = U256::from_dec_str( + &web3_tx_dao + .max_fee_per_gas + .clone() + .ok_or(err_custom_create!("max_fee_per_gas has to be set here"))?, + ) + .map_err(err_from!())?; let gas_needed_for_tx = U256::from_dec_str(&web3_tx_dao.val).map_err(err_from!())?; let maximum_gas_needed = gas_needed_for_tx + gas_limit * max_fee_per_gas; Ok(maximum_gas_needed) From 35648f5c0d4aded8fc0120af942e58946c6b3dd6 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 24 Oct 2023 17:05:39 +0200 Subject: [PATCH 03/10] remove signed_raw_data after transaction confirmed (it's just unneeded junk at this point) --- crates/erc20_payment_lib/src/sender/process.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/erc20_payment_lib/src/sender/process.rs b/crates/erc20_payment_lib/src/sender/process.rs index a7517f31..1e5bd802 100644 --- a/crates/erc20_payment_lib/src/sender/process.rs +++ b/crates/erc20_payment_lib/src/sender/process.rs @@ -523,6 +523,8 @@ pub async fn process_transaction( } } current_tx.orig_tx_id = None; + //signed raw data is no longer needed + current_tx.signed_raw_data = None; update_tx(&mut *db_transaction, ¤t_tx) .await .map_err(err_from!())?; From be569545c8381398b2c54029d9831688ddd4fd9e Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 24 Oct 2023 18:54:34 +0200 Subject: [PATCH 04/10] working on better gather time handling --- config-payments.toml | 8 +- crates/erc20_payment_lib/src/config.rs | 3 +- crates/erc20_payment_lib/src/runtime.rs | 9 +- .../erc20_payment_lib/src/sender/process.rs | 14 +- .../erc20_payment_lib/src/sender/service.rs | 237 +++++++++--------- crates/erc20_payment_lib/src/setup.rs | 9 +- .../src/account_balance.rs | 2 +- .../src/config_setup.rs | 3 +- src/main.rs | 2 +- src/options.rs | 2 +- 10 files changed, 150 insertions(+), 139 deletions(-) diff --git a/config-payments.toml b/config-payments.toml index 4f091a12..4fdb56b2 100644 --- a/config-payments.toml +++ b/config-payments.toml @@ -1,6 +1,10 @@ [engine] -service-sleep = 10 -process-sleep = 10 +# service sleep is for internal runtime checks +service-sleep = 5 +# process sleep is to set how often payments are gathered +gather-sleep = 600 +# gather payments on payment driver start (otherwise wait for first gather-sleep) +gather-at-start = true automatic-recover = false [chain.rinkeby] diff --git a/crates/erc20_payment_lib/src/config.rs b/crates/erc20_payment_lib/src/config.rs index 09689385..bbafdfa7 100644 --- a/crates/erc20_payment_lib/src/config.rs +++ b/crates/erc20_payment_lib/src/config.rs @@ -47,7 +47,8 @@ impl AdditionalOptions { #[serde(rename_all = "kebab-case")] pub struct Engine { pub service_sleep: u64, - pub process_sleep: u64, + pub gather_sleep: u64, + pub gather_at_start: bool, pub automatic_recover: bool, } diff --git a/crates/erc20_payment_lib/src/runtime.rs b/crates/erc20_payment_lib/src/runtime.rs index 86aa7548..662de5da 100644 --- a/crates/erc20_payment_lib/src/runtime.rs +++ b/crates/erc20_payment_lib/src/runtime.rs @@ -178,7 +178,8 @@ pub struct ValidatedOptions { pub generate_tx_only: bool, pub skip_multi_contract_check: bool, pub service_sleep: u64, - pub process_sleep: u64, + pub gather_sleep: u64, + pub gather_at_start: bool, pub http_threads: u64, pub http_port: u16, pub http_addr: String, @@ -195,7 +196,8 @@ impl Default for ValidatedOptions { generate_tx_only: false, skip_multi_contract_check: false, service_sleep: 10, - process_sleep: 10, + gather_sleep: 10, + gather_at_start: false, http_threads: 2, http_port: 8080, http_addr: "127.0.0.1".to_string(), @@ -398,7 +400,8 @@ impl PaymentRuntime { options.generate_tx_only, options.skip_multi_contract_check, config.engine.service_sleep, - config.engine.process_sleep, + config.engine.gather_sleep, + config.engine.gather_at_start, config.engine.automatic_recover, )?; payment_setup.use_transfer_for_single_payment = options.use_transfer_for_single_payment; diff --git a/crates/erc20_payment_lib/src/sender/process.rs b/crates/erc20_payment_lib/src/sender/process.rs index 1e5bd802..9db11d87 100644 --- a/crates/erc20_payment_lib/src/sender/process.rs +++ b/crates/erc20_payment_lib/src/sender/process.rs @@ -59,7 +59,7 @@ pub async fn process_transaction( ) -> Result<(TxDao, ProcessTransactionResult), PaymentError> { const CHECKS_UNTIL_NOT_FOUND: u64 = 5; - let wait_duration = Duration::from_secs(payment_setup.process_sleep); + let gather_sleep = Duration::from_secs(payment_setup.gather_sleep); let chain_id = web3_tx_dao.chain_id; let Ok(chain_setup) = payment_setup.get_chain_setup(chain_id) else { @@ -754,7 +754,11 @@ pub async fn process_transaction( send_transaction(event_sender.clone(), web3, web3_tx_dao).await?; web3_tx_dao.broadcast_count += 1; update_tx(conn, web3_tx_dao).await.map_err(err_from!())?; - tokio::time::sleep(wait_duration).await; + log::warn!( + "Sleeping for {} seconds (process sleep after transaction send)", + gather_sleep.as_secs() + ); + tokio::time::sleep(gather_sleep).await; continue; } else { //timeout transaction when it is not confirmed after transaction_timeout seconds @@ -831,6 +835,10 @@ pub async fn process_transaction( if !wait_for_confirmation { return Ok((web3_tx_dao.clone(), ProcessTransactionResult::Unknown)); } - tokio::time::sleep(wait_duration).await; + log::warn!( + "Sleeping for {} seconds (process sleep at the end of the loop)", + gather_sleep.as_secs() + ); + tokio::time::sleep(gather_sleep).await; } } diff --git a/crates/erc20_payment_lib/src/sender/service.rs b/crates/erc20_payment_lib/src/sender/service.rs index 18a06f5b..862914a3 100644 --- a/crates/erc20_payment_lib/src/sender/service.rs +++ b/crates/erc20_payment_lib/src/sender/service.rs @@ -2,6 +2,7 @@ use crate::db::model::*; use crate::db::ops::*; use crate::error::{ErrorBag, PaymentError}; use std::sync::Arc; +use std::time::Duration; use tokio::sync::Mutex; use crate::sender::process::{process_transaction, ProcessTransactionResult}; @@ -160,7 +161,6 @@ pub async fn update_approve_result( ProcessTransactionResult::Confirmed => { tx.processing = 0; - log::error!("Updating approve result {tx:?}"); let mut db_transaction = conn.begin().await.map_err(err_from!())?; let mut allowance = get_allowance_by_tx(&mut *db_transaction, tx.id) .await @@ -266,83 +266,83 @@ pub async fn process_transactions( .await .map_err(err_from!())?; - if let Some(tx) = transactions.get_mut(0) { - let (mut tx, process_t_res) = if shared_state.lock().await.is_skipped(tx.id) { - ( - tx.clone(), - ProcessTransactionResult::InternalError("Transaction skipped by user".into()), - ) - } else { - shared_state - .lock() - .await - .set_tx_message(tx.id, "Processing".to_string()); - match process_transaction( - event_sender.clone(), - shared_state.clone(), - conn, - tx, - payment_setup, - signer, - false, - ) - .await - { - Ok((tx_dao, process_result)) => (tx_dao, process_result), - Err(err) => match err.inner { - ErrorBag::TransactionFailedError(err2) => { - shared_state - .lock() - .await - .set_tx_error(tx.id, Some(err2.message.clone())); + let Some(tx) = transactions.get_mut(0) else { + break; + }; - return Err(err_create!(err2)); - } - _ => { - log::error!("Error in process transaction: {}", err.inner); - shared_state - .lock() - .await - .set_tx_error(tx.id, Some(format!("{}", err.inner))); - return Err(err); - } - }, - } - }; - if let ProcessTransactionResult::Replaced = process_t_res { - continue; - }; - if tx.method.starts_with("MULTI.golemTransfer") - || tx.method == "ERC20.transfer" - || tx.method == "transfer" + let (mut tx, process_t_res) = if shared_state.lock().await.is_skipped(tx.id) { + ( + tx.clone(), + ProcessTransactionResult::InternalError("Transaction skipped by user".into()), + ) + } else { + shared_state + .lock() + .await + .set_tx_message(tx.id, "Processing".to_string()); + match process_transaction( + event_sender.clone(), + shared_state.clone(), + conn, + tx, + payment_setup, + signer, + false, + ) + .await { - log::debug!("Updating token transfer result"); - update_token_transfer_result(event_sender.clone(), conn, &mut tx, &process_t_res) - .await?; - } else if tx.method == "ERC20.approve" { - log::debug!("Updating token approve result"); - update_approve_result(event_sender.clone(), conn, &mut tx, &process_t_res).await?; - } else { - log::debug!("Updating plain tx result"); - update_tx_result(conn, &mut tx, &process_t_res).await?; - } - match process_t_res { - ProcessTransactionResult::Unknown => {} - ProcessTransactionResult::Confirmed => { - send_driver_event( - &event_sender, - DriverEventContent::TransactionConfirmed(tx.clone()), - ) - .await; - } - _ => { - shared_state.lock().await.current_tx_info.remove(&tx.id); - } + Ok((tx_dao, process_result)) => (tx_dao, process_result), + Err(err) => match err.inner { + ErrorBag::TransactionFailedError(err2) => { + shared_state + .lock() + .await + .set_tx_error(tx.id, Some(err2.message.clone())); + + return Err(err_create!(err2)); + } + _ => { + log::error!("Error in process transaction: {}", err.inner); + shared_state + .lock() + .await + .set_tx_error(tx.id, Some(format!("{}", err.inner))); + return Err(err); + } + }, } + }; + if let ProcessTransactionResult::Replaced = process_t_res { + continue; + }; + if tx.method.starts_with("MULTI.golemTransfer") + || tx.method == "ERC20.transfer" + || tx.method == "transfer" + { + log::debug!("Updating token transfer result"); + update_token_transfer_result(event_sender.clone(), conn, &mut tx, &process_t_res) + .await?; + } else if tx.method == "ERC20.approve" { + log::debug!("Updating token approve result"); + update_approve_result(event_sender.clone(), conn, &mut tx, &process_t_res).await?; + } else { + log::debug!("Updating plain tx result"); + update_tx_result(conn, &mut tx, &process_t_res).await?; } - if transactions.is_empty() { - break; + match process_t_res { + ProcessTransactionResult::Unknown => {} + ProcessTransactionResult::Confirmed => { + send_driver_event( + &event_sender, + DriverEventContent::TransactionConfirmed(tx.clone()), + ) + .await; + } + _ => { + shared_state.lock().await.current_tx_info.remove(&tx.id); + } } + tokio::time::sleep(std::time::Duration::from_secs(payment_setup.service_sleep)).await; } Ok(()) @@ -355,58 +355,53 @@ pub async fn service_loop( signer: impl Signer + Send + Sync + 'static, event_sender: Option>, ) { - let process_transactions_interval = payment_setup.process_sleep as i64; - let gather_transactions_interval = payment_setup.process_sleep as i64; - let mut last_update_time1 = - chrono::Utc::now() - chrono::Duration::seconds(process_transactions_interval); - let mut last_update_time2 = - chrono::Utc::now() - chrono::Duration::seconds(gather_transactions_interval); + let gather_transactions_interval = payment_setup.gather_sleep as i64; + let mut last_gather_time = if payment_setup.gather_at_start { + chrono::Utc::now() - chrono::Duration::seconds(gather_transactions_interval) + } else { + chrono::Utc::now() + }; - let mut process_tx_needed = true; - let mut process_tx_instantly = true; + let mut process_tx_needed; loop { log::debug!("Sender service loop - start loop"); let current_time = chrono::Utc::now(); - if current_time < last_update_time1 { - //handle case when system time changed - last_update_time1 = current_time; - } - if process_tx_instantly - || (process_tx_needed - && current_time - > last_update_time1 + chrono::Duration::seconds(process_transactions_interval)) + if payment_setup.generate_tx_only { + log::warn!("Skipping processing transactions..."); + } else if let Err(e) = process_transactions( + event_sender.clone(), + shared_state.clone(), + conn, + payment_setup, + &signer, + ) + .await { - process_tx_instantly = false; - if payment_setup.generate_tx_only { - log::warn!("Skipping processing transactions..."); - process_tx_needed = false; - } else { - match process_transactions( - event_sender.clone(), - shared_state.clone(), - conn, - payment_setup, - &signer, - ) - .await - { - Ok(_) => { - //all pending transactions processed - process_tx_needed = false; - } - Err(e) => { - log::error!("Error in process transactions: {}", e); - } - }; - } - last_update_time1 = current_time; + log::error!("Error in process transactions: {}", e); + tokio::time::sleep(std::time::Duration::from_secs(10)).await; + continue; } - if current_time - > last_update_time2 + chrono::Duration::seconds(gather_transactions_interval) - && !process_tx_needed - { + process_tx_needed = false; + + //we should be here only when all pending transactions are processed + + let next_gather_time = + last_gather_time + chrono::Duration::seconds(gather_transactions_interval); + if current_time < next_gather_time { + log::info!( + "Transaction will be gathered in {} seconds", + humantime::format_duration(Duration::from_secs( + (next_gather_time - current_time).num_seconds() as u64 + )) + ); + tokio::time::sleep(Duration::from_secs(std::cmp::min( + 60, + (next_gather_time - current_time).num_seconds() as u64, + ))) + .await; + } else { log::info!("Gathering transfers..."); let mut token_transfer_map = match gather_transactions_pre(conn, payment_setup).await { Ok(token_transfer_map) => token_transfer_map, @@ -429,7 +424,6 @@ pub async fn service_loop( Ok(count) => { if count > 0 { process_tx_needed = true; - process_tx_instantly = true; } else { log::info!("No new transfers to process"); } @@ -443,8 +437,6 @@ pub async fn service_loop( { Ok(_) => { //process transaction instantly - process_tx_needed = true; - process_tx_instantly = true; shared_state.lock().await.idling = false; continue; } @@ -462,7 +454,7 @@ pub async fn service_loop( log::error!("Error in gather transactions: {}", e); } }; - last_update_time2 = current_time; + last_gather_time = current_time; if payment_setup.finish_when_done && !process_tx_needed { log::info!("No more work to do, exiting..."); break; @@ -474,7 +466,6 @@ pub async fn service_loop( shared_state.lock().await.idling = false; } } - - tokio::time::sleep(std::time::Duration::from_secs(payment_setup.service_sleep)).await; + tokio::time::sleep(std::time::Duration::from_secs(1)).await; } } diff --git a/crates/erc20_payment_lib/src/setup.rs b/crates/erc20_payment_lib/src/setup.rs index 34a31393..fe461317 100644 --- a/crates/erc20_payment_lib/src/setup.rs +++ b/crates/erc20_payment_lib/src/setup.rs @@ -61,7 +61,8 @@ pub struct PaymentSetup { pub generate_tx_only: bool, pub skip_multi_contract_check: bool, pub service_sleep: u64, - pub process_sleep: u64, + pub gather_sleep: u64, + pub gather_at_start: bool, pub automatic_recover: bool, pub contract_use_direct_method: bool, pub contract_use_unpacked_method: bool, @@ -78,7 +79,8 @@ impl PaymentSetup { generate_txs_only: bool, skip_multi_contract_check: bool, service_sleep: u64, - process_sleep: u64, + gather_sleep: u64, + gather_at_start: bool, automatic_recover: bool, ) -> Result { let mut ps = PaymentSetup { @@ -89,7 +91,8 @@ impl PaymentSetup { generate_tx_only: generate_txs_only, skip_multi_contract_check, service_sleep, - process_sleep, + gather_sleep, + gather_at_start, automatic_recover, contract_use_direct_method: false, contract_use_unpacked_method: false, diff --git a/crates/erc20_payment_lib_extra/src/account_balance.rs b/crates/erc20_payment_lib_extra/src/account_balance.rs index 6e3a2a77..41c6cb5a 100644 --- a/crates/erc20_payment_lib_extra/src/account_balance.rs +++ b/crates/erc20_payment_lib_extra/src/account_balance.rs @@ -62,7 +62,7 @@ pub async fn account_balance( account_balance_options.chain_name ))?; - let payment_setup = PaymentSetup::new(config, vec![], true, false, false, 1, 1, false)?; + let payment_setup = PaymentSetup::new(config, vec![], true, false, false, 1, 1, false, false)?; let web3 = payment_setup.get_provider(chain_cfg.chain_id)?; diff --git a/crates/erc20_payment_lib_test/src/config_setup.rs b/crates/erc20_payment_lib_test/src/config_setup.rs index 37c0353f..bba7a7b6 100644 --- a/crates/erc20_payment_lib_test/src/config_setup.rs +++ b/crates/erc20_payment_lib_test/src/config_setup.rs @@ -40,8 +40,9 @@ pub async fn create_default_config_setup(proxy_url_base: &str, proxy_key: &str) chain: chain_map, engine: Engine { service_sleep: 1, - process_sleep: 1, + gather_sleep: 1, automatic_recover: false, + gather_at_start: false, }, } } diff --git a/src/main.rs b/src/main.rs index 2525dd43..8c909bed 100644 --- a/src/main.rs +++ b/src/main.rs @@ -292,7 +292,7 @@ async fn main_internal() -> Result<(), PaymentError> { log::info!("Scanning blockchain {}", scan_blockchain_options.chain_name); let payment_setup = - PaymentSetup::new(&config, vec![], true, false, false, 1, 1, false)?; + PaymentSetup::new(&config, vec![], true, false, false, 1, 1, false, false)?; let chain_cfg = config .chain .get(&scan_blockchain_options.chain_name) diff --git a/src/options.rs b/src/options.rs index 1ee73ab4..4a95508e 100644 --- a/src/options.rs +++ b/src/options.rs @@ -43,7 +43,7 @@ pub struct RunOptions { help = "Sleep time between process loops in seconds", default_value = "10" )] - pub process_sleep: u64, + pub gather_sleep: u64, #[structopt(long = "http", help = "Enable http server")] pub http: bool, From d0b17adb6a2765e99bda17f9bdfd2ec0e36ce1c1 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 24 Oct 2023 19:04:39 +0200 Subject: [PATCH 05/10] Added rustmt --- .../src/db/ops/token_transfer_ops.rs | 13 ++--- .../src/db/ops/transfer_in_ops.rs | 22 ++++---- .../erc20_payment_lib/src/sender/batching.rs | 6 ++- .../erc20_payment_lib/src/sender/process.rs | 50 ++++++++++++------- .../erc20_payment_lib/src/sender/service.rs | 7 ++- .../src/blockchain_setup.rs | 3 +- rustfmt.toml | 1 + src/main.rs | 25 ++++++++-- src/stats.rs | 4 +- 9 files changed, 84 insertions(+), 47 deletions(-) create mode 100644 rustfmt.toml diff --git a/crates/erc20_payment_lib/src/db/ops/token_transfer_ops.rs b/crates/erc20_payment_lib/src/db/ops/token_transfer_ops.rs index 24d4800d..a3f6b7f3 100644 --- a/crates/erc20_payment_lib/src/db/ops/token_transfer_ops.rs +++ b/crates/erc20_payment_lib/src/db/ops/token_transfer_ops.rs @@ -376,15 +376,10 @@ pub async fn get_transfer_count( .fetch_one(conn) .await? } else if let Some(receiver) = receiver { - sqlx::query_scalar::<_, i64>( - format!( - r"SELECT COUNT(*) FROM token_transfer WHERE {transfer_filter} AND receiver_addr = $1" - ) - .as_str(), - ) - .bind(receiver) - .fetch_one(conn) - .await? + sqlx::query_scalar::<_, i64>(format!(r"SELECT COUNT(*) FROM token_transfer WHERE {transfer_filter} AND receiver_addr = $1").as_str()) + .bind(receiver) + .fetch_one(conn) + .await? } else { sqlx::query_scalar::<_, i64>( format!(r"SELECT COUNT(*) FROM token_transfer WHERE {transfer_filter}").as_str(), diff --git a/crates/erc20_payment_lib/src/db/ops/transfer_in_ops.rs b/crates/erc20_payment_lib/src/db/ops/transfer_in_ops.rs index bfd5fb7c..128fe81f 100644 --- a/crates/erc20_payment_lib/src/db/ops/transfer_in_ops.rs +++ b/crates/erc20_payment_lib/src/db/ops/transfer_in_ops.rs @@ -11,17 +11,17 @@ pub async fn insert_transfer_in( VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) RETURNING *; ", ) - .bind(&token_transfer.payment_id) - .bind(&token_transfer.from_addr) - .bind(&token_transfer.receiver_addr) - .bind(token_transfer.chain_id) - .bind(&token_transfer.token_addr) - .bind(&token_transfer.token_amount) - .bind(&token_transfer.tx_hash) - .bind(token_transfer.requested_date) - .bind(token_transfer.received_date) - .fetch_one(conn) - .await?; + .bind(&token_transfer.payment_id) + .bind(&token_transfer.from_addr) + .bind(&token_transfer.receiver_addr) + .bind(token_transfer.chain_id) + .bind(&token_transfer.token_addr) + .bind(&token_transfer.token_amount) + .bind(&token_transfer.tx_hash) + .bind(token_transfer.requested_date) + .bind(token_transfer.received_date) + .fetch_one(conn) + .await?; Ok(res) } diff --git a/crates/erc20_payment_lib/src/sender/batching.rs b/crates/erc20_payment_lib/src/sender/batching.rs index 68383603..93c2de97 100644 --- a/crates/erc20_payment_lib/src/sender/batching.rs +++ b/crates/erc20_payment_lib/src/sender/batching.rs @@ -211,7 +211,11 @@ pub async fn gather_transactions_batch_multi( None, )? } else if let Some(multi_contract_address) = chain_setup.multi_contract_address { - log::info!("Inserting transaction stub for ERC20 multi transfer contract: {:?} for {} distinct transfers", multi_contract_address, erc20_to.len()); + log::info!( + "Inserting transaction stub for ERC20 multi transfer contract: {:?} for {} distinct transfers", + multi_contract_address, + erc20_to.len() + ); create_erc20_transfer_multi( Address::from_str(&token_transfer.from_addr).map_err(err_from!())?, diff --git a/crates/erc20_payment_lib/src/sender/process.rs b/crates/erc20_payment_lib/src/sender/process.rs index 9db11d87..805d9721 100644 --- a/crates/erc20_payment_lib/src/sender/process.rs +++ b/crates/erc20_payment_lib/src/sender/process.rs @@ -207,10 +207,11 @@ pub async fn process_transaction( } if new_target_gas_u256 * 11 < max_fee_per_gas * 10 { - log::warn!("Eco mode activated. Sending transaction with lower base fee. Blockchain base fee: {} Gwei, Tx base fee: {} Gwei", + log::warn!( + "Eco mode activated. Sending transaction with lower base fee. Blockchain base fee: {} Gwei, Tx base fee: {} Gwei", blockchain_gas_price.to_gwei().map_err(err_from!())?, new_target_gas_u256.to_gwei().map_err(err_from!())?, - ); + ); max_fee_per_gas = new_target_gas_u256; } @@ -531,7 +532,12 @@ pub async fn process_transaction( db_transaction.commit().await.map_err(err_from!())?; return Ok((current_tx.clone(), ProcessTransactionResult::Confirmed)); } else { - log::info!("Waiting for confirmations: tx: {}. Current block {}, expected at least: {}", web3_tx_dao.id, current_block_number, block_number + chain_setup.confirmation_blocks); + log::info!( + "Waiting for confirmations: tx: {}. Current block {}, expected at least: {}", + web3_tx_dao.id, + current_block_number, + block_number + chain_setup.confirmation_blocks + ); } } else { tx_not_found_count += 1; @@ -645,7 +651,12 @@ pub async fn process_transaction( web3_tx_dao.id ); } else { - log::warn!("Transaction priority fee changed less than 10% more from {} to {} for tx: {}", max_tx_priority_fee_str, chain_setup.priority_fee, web3_tx_dao.id); + log::warn!( + "Transaction priority fee changed less than 10% more from {} to {} for tx: {}", + max_tx_priority_fee_str, + chain_setup.priority_fee, + web3_tx_dao.id + ); } } @@ -806,22 +817,23 @@ pub async fn process_transaction( > tx_max_fee_per_gas_gwei { send_driver_event( - &event_sender, - DriverEventContent::TransactionStuck( - TransactionStuckReason::GasPriceLow( - GasLowInfo { - tx: web3_tx_dao.clone(), - tx_max_fee_per_gas_gwei, - block_date, - block_number: block.number.unwrap().as_u64(), - block_base_fee_per_gas_gwei, - assumed_min_priority_fee_gwei, - user_friendly_message: - format!("Transaction not processed after {} seconds, block base fee per gas + priority fee: {} Gwei is greater than transaction max fee per gas: {} Gwei", chain_setup.transaction_timeout, block_base_fee_per_gas_gwei + assumed_min_priority_fee_gwei, tx_max_fee_per_gas_gwei), - } + &event_sender, + DriverEventContent::TransactionStuck(TransactionStuckReason::GasPriceLow(GasLowInfo { + tx: web3_tx_dao.clone(), + tx_max_fee_per_gas_gwei, + block_date, + block_number: block.number.unwrap().as_u64(), + block_base_fee_per_gas_gwei, + assumed_min_priority_fee_gwei, + user_friendly_message: format!( + "Transaction not processed after {} seconds, block base fee per gas + priority fee: {} Gwei is greater than transaction max fee per gas: {} Gwei", + chain_setup.transaction_timeout, + block_base_fee_per_gas_gwei + assumed_min_priority_fee_gwei, + tx_max_fee_per_gas_gwei ), - )) - .await; + })), + ) + .await; } } } diff --git a/crates/erc20_payment_lib/src/sender/service.rs b/crates/erc20_payment_lib/src/sender/service.rs index 862914a3..bd59a2a8 100644 --- a/crates/erc20_payment_lib/src/sender/service.rs +++ b/crates/erc20_payment_lib/src/sender/service.rs @@ -431,7 +431,12 @@ pub async fn service_loop( Err(e) => { match &e.inner { ErrorBag::NoAllowanceFound(allowance_request) => { - log::info!("No allowance found for contract {} to spend token {} for owner: {}", allowance_request.spender_addr, allowance_request.token_addr, allowance_request.owner); + log::info!( + "No allowance found for contract {} to spend token {} for owner: {}", + allowance_request.spender_addr, + allowance_request.token_addr, + allowance_request.owner + ); match process_allowance(conn, payment_setup, allowance_request, &signer) .await { diff --git a/crates/erc20_payment_lib_test/src/blockchain_setup.rs b/crates/erc20_payment_lib_test/src/blockchain_setup.rs index bc2e67e6..f64bd8f6 100644 --- a/crates/erc20_payment_lib_test/src/blockchain_setup.rs +++ b/crates/erc20_payment_lib_test/src/blockchain_setup.rs @@ -320,7 +320,8 @@ impl GethContainer { "GLM_CONTRACT_ADDRESS=0xfff17584d526aba263025eE7fEF517E4A31D4246".to_string(), "FAUCET_ACCOUNT_PUBLIC_ADDRESS=0xafca53fc9628F0E7603bb2bf8E75F07Ee6442cE6".to_string(), "MAIN_ACCOUNT_PUBLIC_ADDRESS=0x4D6947E072C1Ac37B64600B885772Bd3f27D3E91".to_string(), - "FAUCET_ACCOUNT_PRIVATE_KEY=078d8f6c16446cdb8efbee80535ce8cb32d5b69563bca33e5e6bc0f13f0666b3".to_string()]; + "FAUCET_ACCOUNT_PRIVATE_KEY=078d8f6c16446cdb8efbee80535ce8cb32d5b69563bca33e5e6bc0f13f0666b3".to_string(), + ]; let (web3_proxy_port, geth_rpc_port) = if let (Some(web3_proxy_port), Some(geth_rpc_port)) = (opt.web3_proxy_port, opt.web3_port) diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 00000000..d100efd0 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1 @@ +max_width = 100 \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 8c909bed..e8cdcb06 100644 --- a/src/main.rs +++ b/src/main.rs @@ -353,11 +353,23 @@ async fn main_internal() -> Result<(), PaymentError> { } if scan_info.last_block > start_block { - log::info!("Start block from db is higher than start block from cli {}, using start block from db {}", start_block, scan_info.last_block); + log::info!( + "Start block from db is higher than start block from cli {}, using start block from db {}", + start_block, + scan_info.last_block + ); start_block = scan_info.last_block; } else if scan_info.last_block != -1 { - log::error!("There is old entry in db, remove it to start new scan or give proper block range: start block: {}, last block {}", start_block, scan_info.last_block); - return Err(err_custom_create!("There is old entry in db, remove it to start new scan or give proper block range: start block: {}, last block {}", start_block, scan_info.last_block)); + log::error!( + "There is old entry in db, remove it to start new scan or give proper block range: start block: {}, last block {}", + start_block, + scan_info.last_block + ); + return Err(err_custom_create!( + "There is old entry in db, remove it to start new scan or give proper block range: start block: {}, last block {}", + start_block, + scan_info.last_block + )); } let mut end_block = @@ -369,7 +381,12 @@ async fn main_internal() -> Result<(), PaymentError> { if let Some(blocks_behind) = scan_blockchain_options.blocks_behind { if end_block > current_block - blocks_behind as i64 { - log::info!("End block {} is too close to current block {}, using current block - blocks_behind: {}", end_block, current_block, current_block - blocks_behind as i64); + log::info!( + "End block {} is too close to current block {}, using current block - blocks_behind: {}", + end_block, + current_block, + current_block - blocks_behind as i64 + ); end_block = current_block - blocks_behind as i64; } } diff --git a/src/stats.rs b/src/stats.rs index 93b69e80..86b535ef 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -99,7 +99,9 @@ pub async fn export_stats( .bind(format!("{:.6}", fee_paid_total)) .bind(1.to_string()) .bind(transaction_ids.get(&tx.id).unwrap().len().to_string()) - .execute(&export_conn).await.map_err(err_from!())?; + .execute(&export_conn) + .await + .map_err(err_from!())?; } export_conn.close().await; From 55c3f9f034907f546bc8d194b71e4e902e70b1e3 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 24 Oct 2023 19:10:44 +0200 Subject: [PATCH 06/10] Version 3 --- Cargo.lock | 6 +++--- Cargo.toml | 6 +++--- crates/erc20_payment_lib/Cargo.toml | 2 +- crates/erc20_payment_lib_extra/Cargo.toml | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bc713ba6..cbea6a61 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -938,7 +938,7 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "erc20_payment_lib" -version = "0.2.2" +version = "0.3.0" dependencies = [ "actix-files", "actix-web", @@ -968,7 +968,7 @@ dependencies = [ [[package]] name = "erc20_payment_lib_extra" -version = "0.2.1" +version = "0.3.0" dependencies = [ "actix-files", "actix-web", @@ -1039,7 +1039,7 @@ dependencies = [ [[package]] name = "erc20_processor" -version = "0.2.2" +version = "0.3.0" dependencies = [ "actix-cors", "actix-files", diff --git a/Cargo.toml b/Cargo.toml index d7355aa4..a80fe7b4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ path = "src/main.rs" name = "erc20_processor" description = "Payment processor for ERC20 tokens" authors = ["Sieciech Czajka "] -version = "0.2.2" +version = "0.3.0" edition = "2021" license = "MIT" @@ -99,8 +99,8 @@ eth-keystore = { workspace = true } stream-rate-limiter = { workspace = true } fastrand = { workspace = true } -erc20_payment_lib = { path = "crates/erc20_payment_lib", version = "0.2.1" } -erc20_payment_lib_extra = { path = "crates/erc20_payment_lib_extra", version = "0.2.1" } +erc20_payment_lib = { path = "crates/erc20_payment_lib", version = "0.3.0" } +erc20_payment_lib_extra = { path = "crates/erc20_payment_lib_extra", version = "0.3.0" } [dev-dependencies] bollard = { workspace = true } diff --git a/crates/erc20_payment_lib/Cargo.toml b/crates/erc20_payment_lib/Cargo.toml index 78f5de46..3e9251c7 100644 --- a/crates/erc20_payment_lib/Cargo.toml +++ b/crates/erc20_payment_lib/Cargo.toml @@ -2,7 +2,7 @@ name = "erc20_payment_lib" description = "Payment processor for ERC20 tokens" authors = ["Sieciech Czajka "] -version = "0.2.2" +version = "0.3.0" edition = "2021" license = "MIT" diff --git a/crates/erc20_payment_lib_extra/Cargo.toml b/crates/erc20_payment_lib_extra/Cargo.toml index 4f120b90..93094b0b 100644 --- a/crates/erc20_payment_lib_extra/Cargo.toml +++ b/crates/erc20_payment_lib_extra/Cargo.toml @@ -2,7 +2,7 @@ name = "erc20_payment_lib_extra" description = "Payment processor for ERC20 tokens" authors = ["Sieciech Czajka "] -version = "0.2.1" +version = "0.3.0" edition = "2021" license = "MIT" @@ -35,4 +35,4 @@ actix-files = { workspace = true } serde_json = { workspace = true } uuid = { workspace = true } -erc20_payment_lib = { path = "../../crates/erc20_payment_lib", version = "0.2.0" } +erc20_payment_lib = { path = "../../crates/erc20_payment_lib", version = "0.3.0" } From 1f41c3eeea07b67988f0079e09faf0a05244ee96 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Wed, 25 Oct 2023 12:54:54 +0200 Subject: [PATCH 07/10] f --- crates/erc20_payment_lib/src/sender/batching.rs | 14 ++++++++++++-- crates/erc20_payment_lib/src/sender/process.rs | 2 +- crates/erc20_payment_lib/src/setup.rs | 5 ----- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/crates/erc20_payment_lib/src/sender/batching.rs b/crates/erc20_payment_lib/src/sender/batching.rs index 93c2de97..3f70620c 100644 --- a/crates/erc20_payment_lib/src/sender/batching.rs +++ b/crates/erc20_payment_lib/src/sender/batching.rs @@ -110,7 +110,13 @@ pub async fn gather_transactions_batch_multi( multi_order_vector: &mut [TokenTransferMultiOrder], token_transfer: &TokenTransferMultiKey, ) -> Result { - let chain_setup = payment_setup.get_chain_setup(token_transfer.chain_id)?; + let chain_setup = payment_setup + .chain_setup + .get(&token_transfer.chain_id) + .ok_or(err_custom_create!( + "No setup found for chain id: {}", + token_transfer.chain_id + ))?; let use_direct_method = payment_setup.contract_use_direct_method; let use_unpacked_method = payment_setup.contract_use_unpacked_method; @@ -269,7 +275,11 @@ pub async fn gather_transactions_batch( sum += U256::from_dec_str(&token_transfer.token_amount).map_err(err_from!())?; } - let Ok(_chain_setup) = payment_setup.get_chain_setup(token_transfer.chain_id) else { + if payment_setup + .chain_setup + .get(&token_transfer.chain_id) + .is_none() + { send_driver_event( &event_sender, DriverEventContent::TransactionFailed(TransactionFailedReason::InvalidChainId( diff --git a/crates/erc20_payment_lib/src/sender/process.rs b/crates/erc20_payment_lib/src/sender/process.rs index 805d9721..d8afc599 100644 --- a/crates/erc20_payment_lib/src/sender/process.rs +++ b/crates/erc20_payment_lib/src/sender/process.rs @@ -62,7 +62,7 @@ pub async fn process_transaction( let gather_sleep = Duration::from_secs(payment_setup.gather_sleep); let chain_id = web3_tx_dao.chain_id; - let Ok(chain_setup) = payment_setup.get_chain_setup(chain_id) else { + let Some(chain_setup) = payment_setup.chain_setup.get(&chain_id) else { send_driver_event( &event_sender, DriverEventContent::TransactionFailed(TransactionFailedReason::InvalidChainId( diff --git a/crates/erc20_payment_lib/src/setup.rs b/crates/erc20_payment_lib/src/setup.rs index fe461317..74258170 100644 --- a/crates/erc20_payment_lib/src/setup.rs +++ b/crates/erc20_payment_lib/src/setup.rs @@ -162,11 +162,6 @@ impl PaymentSetup { } Ok(ps) } - pub fn get_chain_setup(&self, chain_id: i64) -> Result<&ChainSetup, PaymentError> { - self.chain_setup - .get(&chain_id) - .ok_or_else(|| err_custom_create!("No chain setup for chain id: {}", chain_id)) - } pub fn get_provider(&self, chain_id: i64) -> Result<&Web3, PaymentError> { let chain_setup = self From 49521432a1794edca38f3cd834cd3856202d98d6 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Wed, 25 Oct 2023 12:56:39 +0200 Subject: [PATCH 08/10] f --- crates/erc20_payment_lib/src/runtime.rs | 37 ------------------------- 1 file changed, 37 deletions(-) diff --git a/crates/erc20_payment_lib/src/runtime.rs b/crates/erc20_payment_lib/src/runtime.rs index 662de5da..764dacdf 100644 --- a/crates/erc20_payment_lib/src/runtime.rs +++ b/crates/erc20_payment_lib/src/runtime.rs @@ -168,43 +168,6 @@ impl SharedState { } } -#[derive(Clone)] -pub struct ValidatedOptions { - pub receivers: Vec
, - pub amounts: Vec, - pub chain_id: i64, - pub token_addr: Option
, - pub keep_running: bool, - pub generate_tx_only: bool, - pub skip_multi_contract_check: bool, - pub service_sleep: u64, - pub gather_sleep: u64, - pub gather_at_start: bool, - pub http_threads: u64, - pub http_port: u16, - pub http_addr: String, -} - -impl Default for ValidatedOptions { - fn default() -> Self { - ValidatedOptions { - receivers: vec![], - amounts: vec![], - chain_id: 80001, - token_addr: None, - keep_running: true, - generate_tx_only: false, - skip_multi_contract_check: false, - service_sleep: 10, - gather_sleep: 10, - gather_at_start: false, - http_threads: 2, - http_port: 8080, - http_addr: "127.0.0.1".to_string(), - } - } -} - #[derive(Clone, Debug, PartialEq, Serialize)] pub enum StatusProperty { InvalidChainId { From 16e935a18f395cabbb0e35a6b1e840f1fc684ce5 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Wed, 25 Oct 2023 12:59:30 +0200 Subject: [PATCH 09/10] f --- crates/erc20_payment_lib/src/sender/process.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/erc20_payment_lib/src/sender/process.rs b/crates/erc20_payment_lib/src/sender/process.rs index d8afc599..8e2c0ec7 100644 --- a/crates/erc20_payment_lib/src/sender/process.rs +++ b/crates/erc20_payment_lib/src/sender/process.rs @@ -144,12 +144,12 @@ pub async fn process_transaction( } } let mut max_fee_per_gas = if let Some(max_fee_per_gas) = &web3_tx_dao.max_fee_per_gas { - U256::from_dec_str(max_fee_per_gas).map_err(err_from!())? + max_fee_per_gas.to_u256().map_err(err_from!())? } else { chain_setup.max_fee_per_gas }; let max_priority_fee = if let Some(priority_fee) = &web3_tx_dao.priority_fee { - U256::from_dec_str(priority_fee).map_err(err_from!())? + priority_fee.to_u256().map_err(err_from!())? } else { chain_setup.priority_fee }; From b914aba0888a82780510925dc413d59dba3e7e5f Mon Sep 17 00:00:00 2001 From: scx1332 Date: Wed, 25 Oct 2023 13:03:06 +0200 Subject: [PATCH 10/10] f --- crates/erc20_payment_lib/src/sender/process.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/erc20_payment_lib/src/sender/process.rs b/crates/erc20_payment_lib/src/sender/process.rs index 8e2c0ec7..2eafab5b 100644 --- a/crates/erc20_payment_lib/src/sender/process.rs +++ b/crates/erc20_payment_lib/src/sender/process.rs @@ -199,11 +199,10 @@ pub async fn process_transaction( } new_target_gas_u256 }; - let tx_priority_fee_u256 = max_priority_fee; //max_fee_per_gas cannot be lower than priority fee - if new_target_gas_u256 < tx_priority_fee_u256 { - new_target_gas_u256 = tx_priority_fee_u256; + if new_target_gas_u256 < max_priority_fee { + new_target_gas_u256 = max_priority_fee; } if new_target_gas_u256 * 11 < max_fee_per_gas * 10 {