From 91f6a5cc2e0dc4b83cca0b35e6a1240c59e7cac8 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Thu, 16 Nov 2023 17:25:12 +0100 Subject: [PATCH] working on test loop --- .github/workflows/release.yml | 336 +++++++++--------- .../erc20_payment_lib/src/account_balance.rs | 214 +++-------- crates/erc20_payment_lib/src/runtime.rs | 30 +- crates/erc20_payment_lib_extra/src/lib.rs | 3 +- .../erc20_payment_lib_test/src/get_balance.rs | 5 +- crates/web3_test_proxy/Cargo.toml | 1 - crates/web3_test_proxy/src/main.rs | 109 +++--- crates/web3_test_proxy/src/plan.rs | 178 +++++----- crates/web3_test_proxy/src/problems.rs | 11 +- src/main.rs | 5 +- src/options.rs | 3 +- 11 files changed, 389 insertions(+), 506 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 19a14e25..710de94a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,168 +1,168 @@ -name: Release -on: - push: - tags: - - "v*.*.*" - -permissions: - packages: write - contents: write - -jobs: - create-release: - name: Create Release - runs-on: ubuntu-latest - steps: - - name: Release - uses: softprops/action-gh-release@v1 - - frontend: - name: Build frontend - timeout-minutes: 20 - continue-on-error: true - needs: create-release - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Build frontend - run: | - cd frontend - npm install - npm run build - - - name: Pack assets - run: | - tar -cf - frontend/dist | xz -9 > frontend.tar.xz - - - name: Upload - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: frontend.tar.xz - asset_name: frontend.tar.xz - tag: ${{ github.ref }} - overwrite: true - body: "Release ${{ github.ref }}" - - build: - runs-on: ${{ matrix.build-on }} - continue-on-error: true - needs: create-release - strategy: - matrix: - include: - - cpu: x86_64 - os: win - target: x86_64-pc-windows-msvc - build-on: windows-latest - build-with: cargo - exe: ".exe" - run-tests: true - - cpu: x86_64 - os: linux - target: x86_64-unknown-linux-musl - build-on: ubuntu-latest - build-with: cargo - exe: "" - run-tests: true - - cpu: aarch64 - os: linux - target: aarch64-unknown-linux-musl - build-on: ubuntu-latest - build-with: cross - exe: "" - run-tests: false - - cpu: aarch64 - os: macOS - target: aarch64-apple-darwin - build-on: macos-latest - build-with: cargo - exe: "" - run-tests: false - - cpu: x86_64 - os: macOS - target: x86_64-apple-darwin - build-on: macos-latest - build-with: cargo - exe: "" - run-tests: true - - name: Build Release ${{ matrix.cpu }} ${{ matrix.os }} - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Extract version - id: version - run: | - echo version=${GITHUB_REF#refs/*/} >> $GITHUB_OUTPUT - - - name: Update musl tools - if: matrix.build-with == 'cargo' && matrix.os == 'linux' - run: | - sudo apt update - sudo apt install -y musl-tools - - - name: Update Rust and add toolchain ${{ matrix.target }} - if: matrix.build-with == 'cargo' - run: | - rustup update - rustup target add ${{ matrix.target }} - - - name: Install bin install if needed - if: matrix.build-with == 'cross' - run: | - wget -qO- https://github.com/cargo-bins/cargo-binstall/releases/latest/download/cargo-binstall-x86_64-unknown-linux-musl.tgz | tar xvz -C ~/.cargo/bin - - - name: Install cross if needed - if: matrix.build-with == 'cross' - run: | - cargo binstall cross -y - - - name: Build binary target/${{ matrix.target }}/release/erc20_processor - run: | - ${{ matrix.build-with }} build --profile release-lto --target ${{ matrix.target }} - - - name: Create and push docker image - if: matrix.os == 'linux' && matrix.cpu == 'x86_64' - run: | - ${{ matrix.build-with }} build -p web3_test_proxy --profile release-lto --target ${{ matrix.target }} - cp target/${{ matrix.target }}/release-lto/erc20_processor build/docker/erc20_processor - cp target/${{ matrix.target }}/release-lto/web3_test_proxy build/docker/web3_test_proxy - - # login to ghcr.io - docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} - - # build with full metadata - docker build \ - --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ - --label "org.opencontainers.image.description=Erc20 payment processor binary in docker alpine" \ - --label "org.opencontainers.image.licenses=MIT" \ - -t ghcr.io/golemfactory/erc20_processor:latest \ - build/docker - - # tag image with the same tag as the release - docker tag \ - ghcr.io/golemfactory/erc20_processor:latest \ - ghcr.io/golemfactory/erc20_processor:${{ steps.version.outputs.version }} - - # push one image with two tags into repository - docker push --all-tags ghcr.io/golemfactory/erc20_processor - - - name: Compress binaries - run: | - # mv target/${{ matrix.target }}/release-lto/erc20_processor${{ matrix.exe }} target/${{ matrix.target }}/release-lto/erc20_processor${{ matrix.exe }} - tar -cf - -C target/${{ matrix.target }}/release-lto/ erc20_processor${{ matrix.exe }} | xz -9 > erc20_processor.tar.xz - - - name: Upload - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: erc20_processor.tar.xz - asset_name: erc20_processor-${{ matrix.os }}-${{ matrix.cpu }}.tar.xz - tag: ${{ github.ref }} - overwrite: true - body: "Release ${{ steps.version.outputs.version }}" +name: Release +on: + push: + tags: + - "v*.*.*" + +permissions: + packages: write + contents: write + +jobs: + create-release: + name: Create Release + runs-on: ubuntu-latest + steps: + - name: Release + uses: softprops/action-gh-release@v1 + + frontend: + name: Build frontend + timeout-minutes: 20 + continue-on-error: true + needs: create-release + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Build frontend + run: | + cd frontend + npm install + npm run build + + - name: Pack assets + run: | + tar -cf - frontend/dist | xz -9 > frontend.tar.xz + + - name: Upload + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: frontend.tar.xz + asset_name: frontend.tar.xz + tag: ${{ github.ref }} + overwrite: true + body: "Release ${{ github.ref }}" + + build: + runs-on: ${{ matrix.build-on }} + continue-on-error: true + needs: create-release + strategy: + matrix: + include: + - cpu: x86_64 + os: win + target: x86_64-pc-windows-msvc + build-on: windows-latest + build-with: cargo + exe: ".exe" + run-tests: true + - cpu: x86_64 + os: linux + target: x86_64-unknown-linux-musl + build-on: ubuntu-latest + build-with: cargo + exe: "" + run-tests: true + - cpu: aarch64 + os: linux + target: aarch64-unknown-linux-musl + build-on: ubuntu-latest + build-with: cross + exe: "" + run-tests: false + - cpu: aarch64 + os: macOS + target: aarch64-apple-darwin + build-on: macos-latest + build-with: cargo + exe: "" + run-tests: false + - cpu: x86_64 + os: macOS + target: x86_64-apple-darwin + build-on: macos-latest + build-with: cargo + exe: "" + run-tests: true + + name: Build Release ${{ matrix.cpu }} ${{ matrix.os }} + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Extract version + id: version + run: | + echo version=${GITHUB_REF#refs/*/} >> $GITHUB_OUTPUT + + - name: Update musl tools + if: matrix.build-with == 'cargo' && matrix.os == 'linux' + run: | + sudo apt update + sudo apt install -y musl-tools + + - name: Update Rust and add toolchain ${{ matrix.target }} + if: matrix.build-with == 'cargo' + run: | + rustup update + rustup target add ${{ matrix.target }} + + - name: Install bin install if needed + if: matrix.build-with == 'cross' + run: | + wget -qO- https://github.com/cargo-bins/cargo-binstall/releases/latest/download/cargo-binstall-x86_64-unknown-linux-musl.tgz | tar xvz -C ~/.cargo/bin + + - name: Install cross if needed + if: matrix.build-with == 'cross' + run: | + cargo binstall cross -y + + - name: Build binary target/${{ matrix.target }}/release/erc20_processor + run: | + ${{ matrix.build-with }} build --profile release-lto --target ${{ matrix.target }} + + - name: Create and push docker image + if: matrix.os == 'linux' && matrix.cpu == 'x86_64' + run: | + ${{ matrix.build-with }} build -p web3_test_proxy --profile release-lto --target ${{ matrix.target }} + cp target/${{ matrix.target }}/release-lto/erc20_processor build/docker/erc20_processor + cp target/${{ matrix.target }}/release-lto/web3_test_proxy build/docker/web3_test_proxy + + # login to ghcr.io + docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} + + # build with full metadata + docker build \ + --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ + --label "org.opencontainers.image.description=Erc20 payment processor binary in docker alpine" \ + --label "org.opencontainers.image.licenses=MIT" \ + -t ghcr.io/golemfactory/erc20_processor:latest \ + build/docker + + # tag image with the same tag as the release + docker tag \ + ghcr.io/golemfactory/erc20_processor:latest \ + ghcr.io/golemfactory/erc20_processor:${{ steps.version.outputs.version }} + + # push one image with two tags into repository + docker push --all-tags ghcr.io/golemfactory/erc20_processor + + - name: Compress binaries + run: | + # mv target/${{ matrix.target }}/release-lto/erc20_processor${{ matrix.exe }} target/${{ matrix.target }}/release-lto/erc20_processor${{ matrix.exe }} + tar -cf - -C target/${{ matrix.target }}/release-lto/ erc20_processor${{ matrix.exe }} | xz -9 > erc20_processor.tar.xz + + - name: Upload + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: erc20_processor.tar.xz + asset_name: erc20_processor-${{ matrix.os }}-${{ matrix.cpu }}.tar.xz + tag: ${{ github.ref }} + overwrite: true + body: "Release ${{ steps.version.outputs.version }}" diff --git a/crates/erc20_payment_lib/src/account_balance.rs b/crates/erc20_payment_lib/src/account_balance.rs index c39fa4cc..e070e181 100644 --- a/crates/erc20_payment_lib/src/account_balance.rs +++ b/crates/erc20_payment_lib/src/account_balance.rs @@ -1,21 +1,18 @@ +use crate::config::Chain; +use crate::err_from; +use crate::error::ErrorBag; use crate::error::PaymentError; -use crate::eth::get_balance; use crate::runtime::SharedState; use crate::setup::PaymentSetup; -use crate::utils::U256ConvExt; -use crate::{config, err_custom_create}; -use erc20_rpc_pool::{Web3RpcParams, Web3RpcPool}; -use futures_util::{stream, StreamExt}; use serde::{Deserialize, Serialize}; -use std::collections::{BTreeMap, HashSet}; use std::str::FromStr; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; use structopt::StructOpt; use web3::types::Address; #[derive(Clone, StructOpt)] #[structopt(about = "Payment statistics options")] -pub struct BalanceOptions { +pub struct BalanceOptions2 { #[structopt(short = "c", long = "chain-name", default_value = "mumbai")] pub chain_name: String, @@ -47,7 +44,7 @@ pub struct BalanceOptions { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct BalanceResult { +pub struct BalanceResult2 { pub gas: Option, pub gas_decimal: Option, pub gas_human: Option, @@ -56,83 +53,13 @@ pub struct BalanceResult { pub token_human: Option, } -pub async fn account_balance( +pub async fn test_balance_loop( _shared_state: Option>>, - payment_setup: Option, - account_balance_options: BalanceOptions, - config: &config::Config, -) -> Result, PaymentError> { - let chain_cfg = config - .chain - .get(&account_balance_options.chain_name) - .ok_or(err_custom_create!( - "Chain {} not found in config file", - account_balance_options.chain_name - ))?; - - config - .chain - .get(&account_balance_options.chain_name) - .ok_or(err_custom_create!( - "Chain {} not found in config file", - account_balance_options.chain_name - ))?; - let web3_pool = if let Some(ps) = payment_setup { - ps.get_provider(chain_cfg.chain_id).unwrap() - } else { - let web3_pool = Arc::new(Web3RpcPool::new( - chain_cfg.chain_id as u64, - chain_cfg - .rpc_endpoints - .iter() - .map(|rpc| Web3RpcParams { - chain_id: chain_cfg.chain_id as u64, - endpoint: rpc.endpoint.clone(), - skip_validation: rpc.skip_validation.unwrap_or(false), - backup_level: rpc.backup_level.unwrap_or(0), - name: rpc.name.clone(), - verify_interval_secs: rpc.verify_interval_secs.unwrap_or(120), - max_response_time_ms: rpc.max_timeout_ms.unwrap_or(10000), - max_head_behind_secs: rpc.allowed_head_behind_secs, - max_number_of_consecutive_errors: rpc.max_consecutive_errors.unwrap_or(5), - min_interval_requests_ms: rpc.min_interval_ms, - }) - .collect(), - )); - web3_pool - }; - - //shared_state. - //Vec::new(RwLock::new()); - - let token = if !account_balance_options.hide_token { - Some(chain_cfg.token.address) - } else { - None - }; - - //deduplicate accounts using hashset - let accounts = HashSet::::from_iter( - account_balance_options - .accounts - .clone() - .unwrap() - .split(',') - .map(|s| s.trim().to_lowercase()), - ); - - let result_map = Arc::new(Mutex::new(BTreeMap::::new())); - let result_map_ = result_map.clone(); - let mut jobs = Vec::new(); - for account in accounts { - let addr = Address::from_str(&account).map_err(|_| { - err_custom_create!( - "Invalid account address: {}", - account_balance_options.accounts.clone().unwrap() - ) - })?; - jobs.push(addr); - } + payment_setup: PaymentSetup, + account_balance_options: BalanceOptions2, + chain_cfg: &Chain, +) -> Result<(), PaymentError> { + let web3_pool = payment_setup.get_provider(chain_cfg.chain_id).unwrap(); let mut number_of_loops = account_balance_options.debug_loop.unwrap_or(1); if number_of_loops == 0 { @@ -140,95 +67,40 @@ pub async fn account_balance( } let mut prev_loop_time = std::time::Instant::now(); - for i in 0..number_of_loops { - let jobs = jobs.clone(); - - if number_of_loops > 1 { - log::info!("Getting balance: Loop number {}/{}", i, number_of_loops); - if let Some(interval) = account_balance_options.interval { - if i > 0 { - let elapsed = prev_loop_time.elapsed(); - if elapsed.as_secs_f64() < interval { - tokio::time::sleep(std::time::Duration::from_secs_f64( - interval - elapsed.as_secs_f64(), - )) - .await; - } - prev_loop_time = std::time::Instant::now(); - } + let mut job_no = 0_u64; + loop { + if job_no >= number_of_loops { + break; + } + log::info!("Getting balance: Job number {}/{}", job_no, number_of_loops); + if let Some(interval) = account_balance_options.interval { + let elapsed = prev_loop_time.elapsed(); + if elapsed.as_secs_f64() < interval { + tokio::time::sleep(std::time::Duration::from_secs_f64( + interval - elapsed.as_secs_f64(), + )) + .await; } + prev_loop_time = std::time::Instant::now(); } - stream::iter(0..jobs.len()) - .for_each_concurrent(account_balance_options.tasks, |i| { - let job = jobs[i]; - let result_map = result_map_.clone(); - let web3_pool = web3_pool.clone(); - async move { - log::debug!("Getting balance for account: {:#x}", job); - let balance_result = get_balance( - web3_pool.clone(), - token, - job, - !account_balance_options.hide_gas, - ) - .await; - let balance = match balance_result { - Ok(balance) => balance, - Err(err) => { - if number_of_loops > 1 { - log::error!( - "Error getting balance for account: {:#x} - {}", - job, - err - ); - return; - } else { - panic!("Error getting balance for account: {:#x} - {}", job, err); - } - } - }; - - let gas_balance = balance.gas_balance.map(|b| b.to_string()); - let token_balance = balance.token_balance.map(|b| b.to_string()); - log::debug!("{:#x} gas: {:?}", job, gas_balance); - log::debug!("{:#x} token: {:?}", job, token_balance); - let gas_balance_decimal = balance - .gas_balance - .map(|v| v.to_eth().unwrap_or_default().to_string()); - let token_balance_decimal = balance - .token_balance - .map(|v| v.to_eth().unwrap_or_default().to_string()); - let gas_balance_human = gas_balance_decimal.clone().map(|v| { - format!( - "{:.03} {}", - (f64::from_str(&v).unwrap_or(0.0) * 1000.0).floor() / 1000.0, - &chain_cfg.currency_symbol - ) - }); - let token_balance_human = token_balance_decimal.clone().map(|v| { - format!( - "{:.03} {}", - (f64::from_str(&v).unwrap_or(0.0) * 1000.0).floor() / 1000.0, - &chain_cfg.token.symbol - ) - }); - result_map.lock().unwrap().insert( - format!("{:#x}", job), - BalanceResult { - gas: gas_balance, - gas_decimal: gas_balance_decimal, - gas_human: gas_balance_human, - token: token_balance, - token_decimal: token_balance_decimal, - token_human: token_balance_human, - }, - ); - } - }) - .await; + let address = "0x200000000000000000000000".to_string() + + format!("{:#018x}", job_no).replace("0x", "").as_str(); + let address = Address::from_str(&address).map_err(err_from!())?; + + match web3_pool.clone().eth_balance(address, None).await { + Ok(balance) => balance, + Err(err) => { + log::error!( + "Error getting balance for account: {:#x} - {}", + address, + err + ); + continue; + } + }; + job_no += 1; } - let res = result_map.lock().unwrap().clone(); - Ok(res) + Ok(()) } diff --git a/crates/erc20_payment_lib/src/runtime.rs b/crates/erc20_payment_lib/src/runtime.rs index 78ef2107..ea9901ef 100644 --- a/crates/erc20_payment_lib/src/runtime.rs +++ b/crates/erc20_payment_lib/src/runtime.rs @@ -21,7 +21,7 @@ use secp256k1::SecretKey; use sqlx::SqlitePool; use tokio::sync::mpsc::Sender; -use crate::account_balance::{account_balance, BalanceOptions}; +use crate::account_balance::{test_balance_loop, BalanceOptions2}; use crate::config::AdditionalOptions; use crate::db::model::{AllowanceDao, TokenTransferDao, TxDao}; use crate::sender::service_loop; @@ -466,10 +466,14 @@ impl PaymentRuntime { if let Some(balance_check_loop) = extra_testing_.clone().and_then(|e| e.balance_check_loop) { - let balance_options = BalanceOptions { - chain_name: "mumbai".to_string(), + if config_.chain.values().len() != 1 { + panic!("balance_check_loop can be used only with single chain"); + } + let config_chain = config_.chain.values().next().unwrap().clone(); + let balance_options = BalanceOptions2 { + chain_name: "dev".to_string(), //dead address - accounts: Some("0x000000000000000000000000000000000000dead".to_string()), + accounts: Some("0x2000000000000000000000000000000000000000".to_string()), hide_gas: false, hide_token: true, block_number: None, @@ -477,14 +481,22 @@ impl PaymentRuntime { interval: Some(2.0), debug_loop: Some(balance_check_loop), }; - let _ = account_balance( + match test_balance_loop( Some(shared_state_clone), - Some(ps.clone()), + ps.clone(), balance_options, - &config_, + &config_chain, ) - .await; - log::warn!("Balance debug loop finished"); + .await + { + Ok(_) => { + log::info!("Balance debug loop finished"); + } + Err(e) => { + log::error!("Balance debug loop finished with error: {}", e); + panic!("Balance debug loop finished with error: {}", e); + } + } return; } if options.skip_service_loop && options.keep_running { diff --git a/crates/erc20_payment_lib_extra/src/lib.rs b/crates/erc20_payment_lib_extra/src/lib.rs index c7c395c4..926da67d 100644 --- a/crates/erc20_payment_lib_extra/src/lib.rs +++ b/crates/erc20_payment_lib_extra/src/lib.rs @@ -1,4 +1,5 @@ +mod account_balance; mod generate_transactions; -//pub use account_balance::{account_balance, BalanceOptions, BalanceResult}; +pub use account_balance::{account_balance, BalanceOptions, BalanceResult}; pub use generate_transactions::{generate_test_payments, GenerateOptions}; diff --git a/crates/erc20_payment_lib_test/src/get_balance.rs b/crates/erc20_payment_lib_test/src/get_balance.rs index 6aecce44..8560d802 100644 --- a/crates/erc20_payment_lib_test/src/get_balance.rs +++ b/crates/erc20_payment_lib_test/src/get_balance.rs @@ -1,6 +1,6 @@ use crate::config_setup::create_default_config_setup; -use erc20_payment_lib::account_balance::{account_balance, BalanceOptions, BalanceResult}; use erc20_payment_lib::error::PaymentError; +use erc20_payment_lib_extra::{account_balance, BalanceOptions, BalanceResult}; use std::collections::BTreeMap; pub async fn test_get_balance( @@ -16,8 +16,7 @@ pub async fn test_get_balance( block_number: None, tasks: 4, interval: Some(0.001), - debug_loop: None, }; - account_balance(None, None, account_balance_options.clone(), &config_check).await + account_balance(account_balance_options.clone(), &config_check).await } diff --git a/crates/web3_test_proxy/Cargo.toml b/crates/web3_test_proxy/Cargo.toml index 4bd3549a..29d244b1 100644 --- a/crates/web3_test_proxy/Cargo.toml +++ b/crates/web3_test_proxy/Cargo.toml @@ -35,4 +35,3 @@ structopt = { workspace = true } tokio = { workspace = true } toml = { workspace = true } uuid = { workspace = true } -snafu = "0.7.5" diff --git a/crates/web3_test_proxy/src/main.rs b/crates/web3_test_proxy/src/main.rs index 6c464cc9..86bfacee 100644 --- a/crates/web3_test_proxy/src/main.rs +++ b/crates/web3_test_proxy/src/main.rs @@ -1,7 +1,7 @@ mod error; mod frontend; -mod problems; mod plan; +mod problems; extern crate core; @@ -22,9 +22,9 @@ use std::time::{Duration, Instant}; use structopt::StructOpt; use crate::frontend::{frontend_serve, redirect_to_frontend}; +use crate::plan::{ProblemProject, SortedProblemIterator}; use crate::problems::EndpointSimulateProblems; use tokio::sync::Mutex; -use crate::plan::{ProblemProject, SortedProblemIterator}; #[derive(Debug, StructOpt, Clone)] pub struct CliOptions { @@ -66,11 +66,7 @@ pub struct CliOptions { )] pub request_queue_size: usize, - - #[structopt( - long = "problem-plan", - help = "Predefined schedule of problems", - )] + #[structopt(long = "problem-plan", help = "Predefined schedule of problems")] pub problem_plan: Option, } macro_rules! return_on_error_json { @@ -619,7 +615,6 @@ pub async fn get_call(req: HttpRequest, server_data: Data>) -> i })) } - async fn main_internal() -> Result<(), Web3ProxyError> { if let Err(err) = dotenv::dotenv() { log::error!("Cannot load .env file: {err}"); @@ -641,73 +636,75 @@ async fn main_internal() -> Result<(), Web3ProxyError> { #[allow(clippy::manual_map)] let fut = if let Some(problem_plan) = cli.problem_plan.clone() { - Some(tokio::task::spawn( - async move { - let str = std::fs::read(problem_plan).expect("Cannot read problem plan"); - let problem_plan: ProblemProject = serde_json::from_slice(&str).expect("Cannot parse problem plan"); - - let mut problem_project = SortedProblemIterator::from_problem_project(&problem_plan); + Some(tokio::task::spawn(async move { + let str = std::fs::read(problem_plan).expect("Cannot read problem plan"); + let problem_plan: ProblemProject = + serde_json::from_slice(&str).expect("Cannot parse problem plan"); + + let mut problem_project = SortedProblemIterator::from_problem_project(&problem_plan); + + let mut last_time = Instant::now(); + let mut frame_no = 0; + loop { + if let Some(frame_cycle) = problem_plan.frame_cycle { + if frame_no >= frame_cycle { + frame_no = 0; + problem_project = + SortedProblemIterator::from_problem_project(&problem_plan.clone()); + log::info!("Cycle finished, restarting from frame 0"); + } + } + let server_data = server_data_.clone(); - let mut last_time = Instant::now(); - let mut frame_no = 0; loop { - if let Some(frame_cycle) = problem_plan.frame_cycle { - if frame_no >= frame_cycle { - frame_no = 0; - problem_project = SortedProblemIterator::from_problem_project(&problem_plan.clone()); - log::info!("Cycle finished, restarting from frame 0"); - } + if exit_cnd_.load(std::sync::atomic::Ordering::Relaxed) { + return; } - let server_data = server_data_.clone(); - - loop { - if exit_cnd_.load(std::sync::atomic::Ordering::Relaxed) { - return; - } - let sleep_time = problem_plan.frame_interval - last_time.elapsed().as_secs_f64(); - let sleep_time = sleep_time.min(0.1); - if frame_no > 0 && sleep_time > 0.0 { - tokio::time::sleep(Duration::from_secs_f64(sleep_time)).await; - } else { - break; - } + let sleep_time = + problem_plan.frame_interval - last_time.elapsed().as_secs_f64(); + let sleep_time = sleep_time.min(0.1); + if frame_no > 0 && sleep_time > 0.0 { + tokio::time::sleep(Duration::from_secs_f64(sleep_time)).await; + } else { + break; } + } - { - let mut shared_data = server_data.shared_data.lock().await; - while let Some(problem_entry) = problem_project.get_next_entry(frame_no) { - for key in &problem_entry.keys { - let key_data = match shared_data.keys.get_mut(key) { - Some(key_data) => key_data, - None => { - shared_data.keys.insert(key.to_string(), KeyData { + { + let mut shared_data = server_data.shared_data.lock().await; + while let Some(problem_entry) = problem_project.get_next_entry(frame_no) { + for key in &problem_entry.keys { + let key_data = match shared_data.keys.get_mut(key) { + Some(key_data) => key_data, + None => { + shared_data.keys.insert( + key.to_string(), + KeyData { key: key.to_string(), value: "1".to_string(), total_calls: 0, total_requests: 0, calls: VecDeque::new(), problems: EndpointSimulateProblems::default(), - }); - shared_data.keys.get_mut(key).unwrap() - } - }; - key_data.problems.apply_change(&problem_entry.values); - log::info!("Applied change for key: {}, frame: {}", key, frame_no); - } + }, + ); + shared_data.keys.get_mut(key).unwrap() + } + }; + key_data.problems.apply_change(&problem_entry.values); + log::info!("Applied change for key: {}, frame: {}", key, frame_no); } } - - - frame_no += 1; - last_time = Instant::now(); } + + frame_no += 1; + last_time = Instant::now(); } - )) + })) } else { None }; - let server = HttpServer::new(move || { let cors = actix_cors::Cors::default() .allow_any_origin() diff --git a/crates/web3_test_proxy/src/plan.rs b/crates/web3_test_proxy/src/plan.rs index c42127c6..56eeb193 100644 --- a/crates/web3_test_proxy/src/plan.rs +++ b/crates/web3_test_proxy/src/plan.rs @@ -1,89 +1,89 @@ -use std::collections::HashSet; -use serde::{Deserialize, Serialize}; -use crate::problems::ValuesChangeOptions; - - - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ProblemEntry { - frames: Vec, - keys: Vec, - values: ValuesChangeOptions, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ProblemProject { - name : String, - plan_type: String, - pub frame_interval: f64, - pub frame_cycle: Option, - entries: Vec -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SortedProblemIterator { - sorted_entries: Vec, - current_index: usize, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SortedProblemEntry { - frame: u64, - pub keys: Vec, - pub values: ValuesChangeOptions, -} - -impl SortedProblemIterator { - - // sort problems by frame - pub fn from_problem_project(problem_project: &ProblemProject) -> SortedProblemIterator { - let sorted_entries: Vec = problem_project.entries - .iter() - .flat_map(|entry| { // Use flat_map to handle nested structure - entry.frames.iter().map(move |&frame| { // Iterate over frames and map to SortedProblemEntry - SortedProblemEntry { - frame, - keys: entry.keys.clone(), - values: entry.values.clone(), - } - }) - }) - .collect(); // Collect into a Vec - - // Sort the entries by frame. - let mut sorted_entries = sorted_entries; - sorted_entries.sort_by_key(|e| e.frame); - - let mut check_for_conflict = HashSet::<(String, u64)>::new(); - - // Check for conflicts - for entry in &sorted_entries { - for key in &entry.keys { - let key_frame = (key.clone(), entry.frame); - if check_for_conflict.contains(&key_frame) { - panic!("Duplicate key frame: {:?}", key_frame); - } - check_for_conflict.insert(key_frame); - } - } - - SortedProblemIterator{ - sorted_entries, - current_index: 0, - } - } - - pub fn get_next_entry(&mut self, current_frame: u64) -> Option { - if let Some(problem_entry) = self.sorted_entries.get(self.current_index) { - if problem_entry.frame <= current_frame { - self.current_index += 1; - return Some(problem_entry.clone()) - } - None - } else { - None - } - } -} +use crate::problems::ValuesChangeOptions; +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ProblemEntry { + frames: Vec, + keys: Vec, + values: ValuesChangeOptions, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ProblemProject { + name: String, + plan_type: String, + pub frame_interval: f64, + pub frame_cycle: Option, + entries: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SortedProblemIterator { + sorted_entries: Vec, + current_index: usize, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SortedProblemEntry { + frame: u64, + pub keys: Vec, + pub values: ValuesChangeOptions, +} + +impl SortedProblemIterator { + // sort problems by frame + pub fn from_problem_project(problem_project: &ProblemProject) -> SortedProblemIterator { + let sorted_entries: Vec = problem_project + .entries + .iter() + .flat_map(|entry| { + // Use flat_map to handle nested structure + entry.frames.iter().map(move |&frame| { + // Iterate over frames and map to SortedProblemEntry + SortedProblemEntry { + frame, + keys: entry.keys.clone(), + values: entry.values.clone(), + } + }) + }) + .collect(); // Collect into a Vec + + // Sort the entries by frame. + let mut sorted_entries = sorted_entries; + sorted_entries.sort_by_key(|e| e.frame); + + let mut check_for_conflict = HashSet::<(String, u64)>::new(); + + // Check for conflicts + for entry in &sorted_entries { + for key in &entry.keys { + let key_frame = (key.clone(), entry.frame); + if check_for_conflict.contains(&key_frame) { + panic!("Duplicate key frame: {:?}", key_frame); + } + check_for_conflict.insert(key_frame); + } + } + + SortedProblemIterator { + sorted_entries, + current_index: 0, + } + } + + pub fn get_next_entry(&mut self, current_frame: u64) -> Option { + if let Some(problem_entry) = self.sorted_entries.get(self.current_index) { + if problem_entry.frame <= current_frame { + self.current_index += 1; + return Some(problem_entry.clone()); + } + None + } else { + None + } + } +} diff --git a/crates/web3_test_proxy/src/problems.rs b/crates/web3_test_proxy/src/problems.rs index 3d9e0aa9..ecce1ed2 100644 --- a/crates/web3_test_proxy/src/problems.rs +++ b/crates/web3_test_proxy/src/problems.rs @@ -61,11 +61,16 @@ impl EndpointSimulateProblems { if let Some(malformed_response_chance) = change.malformed_response_chance { self.malformed_response_chance = malformed_response_chance; } - if let Some(skip_sending_raw_transaction_chance) = change.skip_sending_raw_transaction_chance { + if let Some(skip_sending_raw_transaction_chance) = + change.skip_sending_raw_transaction_chance + { self.skip_sending_raw_transaction_chance = skip_sending_raw_transaction_chance; } - if let Some(send_transaction_but_report_failure_chance) = change.send_transaction_but_report_failure_chance { - self.send_transaction_but_report_failure_chance = send_transaction_but_report_failure_chance; + if let Some(send_transaction_but_report_failure_chance) = + change.send_transaction_but_report_failure_chance + { + self.send_transaction_but_report_failure_chance = + send_transaction_but_report_failure_chance; } if let Some(allow_only_parsed_calls) = change.allow_only_parsed_calls { self.allow_only_parsed_calls = allow_only_parsed_calls; diff --git a/src/main.rs b/src/main.rs index 7b906af8..e3d42457 100644 --- a/src/main.rs +++ b/src/main.rs @@ -32,9 +32,8 @@ use erc20_payment_lib::runtime::{ use erc20_payment_lib::service::transaction_from_chain_and_into_db; use erc20_payment_lib::setup::PaymentSetup; use erc20_payment_lib::transaction::import_erc20_txs; -use erc20_payment_lib_extra::generate_test_payments; +use erc20_payment_lib_extra::{account_balance, generate_test_payments}; -use erc20_payment_lib::account_balance::account_balance; use erc20_payment_lib::faucet_client::faucet_donate; use erc20_payment_lib::misc::gen_private_keys; use erc20_payment_lib::utils::{DecimalConvExt, StringConvExt}; @@ -488,7 +487,7 @@ async fn main_internal() -> Result<(), PaymentError> { ); } - let result = account_balance(None, None, account_balance_options, &config).await?; + let result = account_balance(account_balance_options, &config).await?; println!( "{}", serde_json::to_string_pretty(&result).map_err(|err| err_custom_create!( diff --git a/src/options.rs b/src/options.rs index 5813004d..b87e2973 100644 --- a/src/options.rs +++ b/src/options.rs @@ -1,7 +1,6 @@ use std::{fmt::Debug, path::PathBuf}; -use erc20_payment_lib::account_balance::BalanceOptions; -use erc20_payment_lib_extra::GenerateOptions; +use erc20_payment_lib_extra::{BalanceOptions, GenerateOptions}; use structopt::StructOpt; use web3::types::Address;