diff --git a/package.json b/package.json index 6ac6e6e7..4fa60cce 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,17 @@ "gas-refund:prod:compute-gas-refund-save-db": "node scripts/gas-refund-program/computeGasRefund.js", "gas-refund:computeDistributionDataAndPersistDB": "patch-package && NODE_ENV=development ts-node scripts/gas-refund-program/distribution/computeDistributionDataAndPersistDB", "gas-refund:computeDistributionDataAndPersistDB-epoch-47": "DISTRIBUTED_EPOCH=47 yarn gas-refund:computeDistributionDataAndPersistDB", + "gas-refund:computeDistributionDataAndPersistDB-epoch-48": "DISTRIBUTED_EPOCH=48 yarn gas-refund:computeDistributionDataAndPersistDB", "gas-refund:computeDistributionFilesAndPersistIPFS": "patch-package && NODE_ENV=development ts-node scripts/gas-refund-program/distribution/computeDistributionFilesAndPersistIPFS", + "gas-refund:generate-dune-query": "patch-package && NODE_ENV=development ts-node scripts/gas-refund-program/generate-dune-query", + "gas-refund:index-dune-transactions": "patch-package && NODE_ENV=development NODE_OPTIONS='--max-old-space-size=8200' ts-node scripts/gas-refund-program/index-dune-transactions", + "gas-refund:prepare-database-for-DuneTransactions-17": "source .env && psql $ROOT_DB_URI -c \"drop database if exists volume_tracker_dune_transactions_epoch_17_aka_47\" && psql $ROOT_DB_URI -c \"create database volume_tracker_dune_transactions_epoch_17_aka_47\"", + "gas-refund:prepare-database-for-DuneTransactions-18": "source .env && psql $ROOT_DB_URI -c \"drop database if exists volume_tracker_dune_transactions_epoch_18_aka_48\" && psql $ROOT_DB_URI -c \"create database volume_tracker_dune_transactions_epoch_18_aka_48\"", + "gas-refund:index-dune-transactions-epoch-015": "source .env && cat ./.vscode/txses-archive/dune-response-15-with-augustus-v6.json | DATABASE_NAME=volume_tracker_dune_transactions_epoch_015_aka_46 DATABASE_URL=$DUNE_TRANSACTIONS_DATABASE_URL_015 yarn gas-refund:index-dune-transactions", + "gas-refund:index-dune-transactions-epoch-17": "source .env && cat ./.vscode/txses-archive/dune-response-17.json | DATABASE_NAME=volume_tracker_dune_transactions_epoch_17_aka_47 DATABASE_URL=$DUNE_TRANSACTIONS_DATABASE_URL_17 yarn gas-refund:index-dune-transactions # it's confusing that the used new-style index here is human-readable, not the index of the epoch", + "gas-refund:index-dune-transactions-epoch-18": "source .env && cat ./.vscode/txses-archive/dune-response-18.json | DATABASE_NAME=volume_tracker_dune_transactions_epoch_18_aka_48 DATABASE_URL=$DUNE_TRANSACTIONS_DATABASE_URL_18 yarn gas-refund:index-dune-transactions", + "gas-refund:dev:dune:compute-gas-refund-save-db-18": "source .env && psql $DATABASE_URL -c 'drop table \"DuneTransactions\"'; yarn _copy-indexed-dune-transactions-18; yarn gas-refund:dev:compute-gas-refund-save-db", + "_copy-indexed-dune-transactions-18": "source .env && /usr/lib/postgresql/16/bin/pg_dump -t '\"DuneTransactions\"' $DUNE_TRANSACTIONS_DATABASE_URL_18 | psql $DATABASE_URL", "migrate:up": "source .env && DATABASE_URL=$DATABASE_URL npx sequelize-cli db:migrate # <- executes any new migrations that are not in sequalize meta table yet, sorted alphabetically", "migrate:undo": "source .env && DATABASE_URL=$DATABASE_URL npx sequelize-cli db:migrate:undo # <- undoes the last migration from sequalize meta table, sorted alphabetically", "test": "jest" diff --git a/scripts/gas-refund-program/generate-dune-query.ts b/scripts/gas-refund-program/generate-dune-query.ts new file mode 100644 index 00000000..901db604 --- /dev/null +++ b/scripts/gas-refund-program/generate-dune-query.ts @@ -0,0 +1,151 @@ +import * as dotenv from 'dotenv'; +dotenv.config(); +import '../../src/lib/log4js'; + +import { + getCurrentEpoch, + resolveEpochCalcTimeInterval, +} from '../../src/lib/gas-refund/epoch-helpers'; +import { GRP_SUPPORTED_CHAINS } from '../../src/lib/gas-refund/gas-refund'; +import { getContractAddresses } from './transactions-indexing/transaction-resolver'; +import * as moment from 'moment'; + +import { MIGRATION_SEPSP2_100_PERCENT_KEY } from './staking/2.0/utils'; +import { isTruthy } from '../../src/lib/utils'; +import { CHAIN_ID_OPTIMISM } from '../../src/lib/constants'; + +export const CHAIN_ID_TO_DUNE_NETWORK: Record = { + 1: 'ethereum', + 56: 'bnb', + 137: 'polygon', + 250: 'fantom', + 10: 'optimism', + 42161: 'arbitrum', + 43114: 'avalanche-c', +}; + +export function timestampToDuneFormatted(timestamp: number) { + return `'${moment.unix(timestamp).utc().format('YYYY-MM-DD HH:mm:ss')}'`; +} + +function getContractsByChainId() { + const currentEpoch = getCurrentEpoch(); + const contractAddressesByChainId = Object.fromEntries( + GRP_SUPPORTED_CHAINS.map(chainId => [ + chainId, + getContractAddresses({ epoch: currentEpoch, chainId }).filter( + address => address !== MIGRATION_SEPSP2_100_PERCENT_KEY, + ), + ]), + ); + return contractAddressesByChainId; +} + +// @TODO: probably should use some tempating engine here +async function generateDuneQuery() { + const currentEpoch = getCurrentEpoch(); + const { startCalcTime, endCalcTime } = await resolveEpochCalcTimeInterval( + currentEpoch - 1, + ); + const dateFrom = timestampToDuneFormatted(startCalcTime); + const dateTo = timestampToDuneFormatted(endCalcTime); + + const conractsByChainId = getContractsByChainId(); + + const parts = GRP_SUPPORTED_CHAINS.map(chainId => { + const network = CHAIN_ID_TO_DUNE_NETWORK[chainId]; + const transactionsInvolvingContract = `transactionsInvolvingContract_${network}`; + const contracts = [...conractsByChainId[chainId]].join(','); + + const txTableColumns = ` +from +gas_price +hash +to +block_number +block_time +gas_used +l1_fee +success +` + .split(/\n/) + .map(s => s.trim()) + .filter(isTruthy); + + const txTableColumnsPart = + chainId === CHAIN_ID_OPTIMISM + ? txTableColumns + .map(s => `cast(transactions."${s}" as varchar) as "${s}"`) + .join(', ') + : txTableColumns + .map(s => + s.includes('l1_') + ? `'n/a' as "${s}"` + : `cast(transactions."${s}" as varchar) as "${s}"`, + ) + .join(', '); + + const networkData = `networkData_${network}`; + const query = ` + + ${transactionsInvolvingContract} as ( + select + tx_hash, + max(to) as contract, + max(block_time), + max(block_number) as block_number + from + ${network}.traces + where + block_time >= to_timestamp(${dateFrom}, 'yyyy-mm-dd hh24:mi:ss') + and block_time <= to_timestamp(${dateTo}, 'yyyy-mm-dd hh24:mi:ss') + and to in (${contracts}) + group by + tx_hash + order by + max(block_time) desc + ), + ${networkData} as ( + select + ${chainId} as chainId, ${transactionsInvolvingContract}.contract as contract, ${txTableColumnsPart} + from + ${transactionsInvolvingContract} + left join ${network}.transactions as transactions on ${transactionsInvolvingContract}.block_number = transactions.block_number + and ${transactionsInvolvingContract}.tx_hash = transactions.hash + and transactions.block_time >= to_timestamp(${dateFrom}, 'yyyy-mm-dd hh24:mi:ss') + and block_time <= to_timestamp(${dateTo}, 'yyyy-mm-dd hh24:mi:ss') + where transactions.success = true + )`; + + return [networkData, query]; + }); + + const queries = parts.map(([, query]) => `${query}`).join(',\n'); + + const unionPart = parts + .map(([networkData]) => `(select * from ${networkData})`) + .join(' UNION \n'); + + return `with ${queries} SELECT * from (\n${unionPart}) ORDER BY block_time DESC`; +} + +async function main() { + const query = await generateDuneQuery(); + console.log('________________________________________________'); + console.log( + "-- This is a generated query. Don't modify it manually, as it'll get overwritten by script", + ); + console.log(query); + console.log('________________________________________________'); + console.log('Use the above output here https://dune.com/queries'); +} + +main() + .then(res => { + console.log('script finished', res); + process.exit(0); + }) + .catch(error => { + console.error('script failed', error); + process.exit(1); + }); diff --git a/scripts/gas-refund-program/index-dune-transactions.ts b/scripts/gas-refund-program/index-dune-transactions.ts new file mode 100644 index 00000000..8c6d9875 --- /dev/null +++ b/scripts/gas-refund-program/index-dune-transactions.ts @@ -0,0 +1,117 @@ +import * as dotenv from 'dotenv'; +dotenv.config(); +import '../../src/lib/log4js'; +import { sliceCalls } from '../../src/lib/utils/helpers'; +import Database from '../../src/database'; +import { DuneRow, DuneTransaction } from '../../src/models/DuneTransaction'; + +import * as fs from 'fs'; + +const tmpDirname = './dune-split-data-temp'; + +// returns tempFolder +function readStdinAndSliceIntoTempFolder() { + var stdinBuffer = fs.readFileSync(0); // STDIN_FILENO = 0 + + const data = JSON.parse(stdinBuffer.toString()); + + console.log('data', data); + + // this is from-browser GQL response + //const origRows = data.data.get_execution.execution_succeeded.data; + + const origRows = data.result.rows; + + fs.mkdirSync(tmpDirname, { recursive: true }); + + sliceCalls({ + inputArray: origRows, + execute: (_rows, sliceIdx) => { + require('fs').writeFileSync( + `${tmpDirname}/slice-${sliceIdx.toString().padStart(3, '0')}.json`, + JSON.stringify(_rows, null, 2), + ); + }, + sliceLength: 10000, + }); + + // this is from-browser GQL response + // const columns = data.data.get_execution.execution_succeeded.columns; + const columns = data.result.metadata.column_names; + require('fs').writeFileSync( + `${tmpDirname}/_columns.json`, + JSON.stringify(columns, null, 2), + ); +} + +function cleanupTempFolder() { + fs.rmSync(tmpDirname, { recursive: true, force: true }); +} + +async function loadSlicesIntoDb() { + const files = fs + .readdirSync(tmpDirname) + .filter(file => !!file.match(/slice-\d+.json/)); + + await Database.connectAndSync('transform-dune-response'); + await DuneTransaction.destroy({ truncate: true }); + + for (let i = 0; i < files.length; i++) { + const filename = files[i]; + const origRows: DuneRow[] = JSON.parse( + fs.readFileSync(`${tmpDirname}/${filename}`).toString(), + ); + + const rows: Partial[] = origRows.map(row => ({ + ...Object.fromEntries( + Object.entries(row).map(([key, value]) => [ + key, + value === 'n/a' ? null : value, + ]), + ), + block_timestamp: row.block_time + ? Date.parse(row.block_time) / 1000 + : undefined, // looks like 2023-08-31 22:14:05.000 UTC + })); + + const queries = sliceCalls({ + inputArray: rows, + execute: async (_rows, sliceIdx) => { + try { + await DuneTransaction.bulkCreate(_rows); + } catch (e) { + // error in bulk? try one by one and find out which row is faulty + try { + await Promise.all( + _rows.map(async row => { + try { + const res = await DuneTransaction.bulkCreate([row]); + } catch (e) { + // to debug in individual level if smth goes wrong + console.error('failed to insert individual', e); + + throw e; + } + }), + ); + } catch (e) { + throw e; + } + throw e; + } + console.log(`inserted slice ${sliceIdx}. Rows length:`, _rows.length); + }, + sliceLength: 1000, + }); + } +} + +async function main() { + readStdinAndSliceIntoTempFolder(); + + await loadSlicesIntoDb(); + + cleanupTempFolder(); +} + +main(); diff --git a/scripts/gas-refund-program/transactions-indexing/fetchRefundableTransactions.ts b/scripts/gas-refund-program/transactions-indexing/fetchRefundableTransactions.ts index 465a1278..8f6789d4 100644 --- a/scripts/gas-refund-program/transactions-indexing/fetchRefundableTransactions.ts +++ b/scripts/gas-refund-program/transactions-indexing/fetchRefundableTransactions.ts @@ -71,7 +71,14 @@ function constructTransactionsProcessor({ `could not retrieve psp/chaincurrency same day rate for swap at ${transaction.timestamp}`, ); - const currGasUsedChainCur = gasSpentInChainCurrencyWei + const currGasUsedChainCur = transaction.txGasUsedUSD // if USD override is available, most likely it's delta -> adjust spent eth and psp to refund based on that + ? new BigNumber( + new BigNumber(transaction.txGasUsedUSD) + .multipliedBy(10 ** 18) + .dividedBy(currencyRate.chainPrice) + .toFixed(0), + ) + : gasSpentInChainCurrencyWei ? new BigNumber(gasSpentInChainCurrencyWei) : new BigNumber(txGasUsed).multipliedBy( transaction.txGasPrice.toString(), @@ -248,24 +255,31 @@ export async function fetchRefundableTransactions({ return result.flat(); }), - ...Array.from(AUGUSTUS_SWAPPERS_V6_OMNICHAIN).map(async contractAddress => { - const epochNewStyle = epoch - GasRefundV2EpochFlip; - - const lastTimestampProcessed = lastTimestampTxByContract[contractAddress]; - - const allStakersTransactionsDuringEpoch = - await fetchParaswapV6StakersTransactions({ - epoch: epochNewStyle, - timestampGreaterThan: lastTimestampProcessed, - chainId, - address: contractAddress, - }); - - return await processRawTxs( - allStakersTransactionsDuringEpoch, - (epoch, totalUserScore) => getRefundPercent(epoch, totalUserScore), - ); - }), + // in this branch v6 txs are sitting together with v5 in global config + ...(chainId != 1 + ? [] // only for chain id=1 take data from metabase + : Array.from(AUGUSTUS_SWAPPERS_V6_OMNICHAIN).map( + async contractAddress => { + const epochNewStyle = epoch - GasRefundV2EpochFlip; + + const lastTimestampProcessed = + lastTimestampTxByContract[contractAddress]; + + const allStakersTransactionsDuringEpoch = + await fetchParaswapV6StakersTransactions({ + epoch: epochNewStyle, + timestampGreaterThan: lastTimestampProcessed, + chainId, + address: contractAddress, + }); + + return await processRawTxs( + allStakersTransactionsDuringEpoch, + (epoch, totalUserScore) => + getRefundPercent(epoch, totalUserScore), + ); + }, + )), ]); const flattened = allTxsV5AndV6Merged.flat(); diff --git a/scripts/gas-refund-program/transactions-indexing/fetchRefundableTransactionsAllChains.ts b/scripts/gas-refund-program/transactions-indexing/fetchRefundableTransactionsAllChains.ts index c6e37f55..7f4e8bb9 100644 --- a/scripts/gas-refund-program/transactions-indexing/fetchRefundableTransactionsAllChains.ts +++ b/scripts/gas-refund-program/transactions-indexing/fetchRefundableTransactionsAllChains.ts @@ -46,7 +46,7 @@ export async function fetchRefundableTransactionsAllChains() { 'cannot compute refund data for epoch < genesis_epoch', ); - for (let epoch = startEpoch; epoch <= getCurrentEpoch(); epoch++) { + for (let epoch = startEpoch; epoch < startEpoch + 1; epoch++) { const { startCalcTime, endCalcTime } = await resolveEpochCalcTimeInterval(epoch); diff --git a/scripts/gas-refund-program/transactions-indexing/transaction-resolver.ts b/scripts/gas-refund-program/transactions-indexing/transaction-resolver.ts index 219f5c74..702b2385 100644 --- a/scripts/gas-refund-program/transactions-indexing/transaction-resolver.ts +++ b/scripts/gas-refund-program/transactions-indexing/transaction-resolver.ts @@ -11,7 +11,11 @@ * is resolved, that is the response of the code in this file. */ import { SPSPAddresses } from '../../../src/lib/staking/spsp-helper'; -import { covalentGetTXsForContractV3 } from './txs-covalent'; +import { + constructCovalentAddressToTransaction, + covalentGetTXsForContractV3, + duneToCovalentLike, +} from './txs-covalent'; import StakesTracker from '../staking/stakes-tracker'; import { getSuccessfulSwaps } from './swaps-subgraph'; import { @@ -32,6 +36,8 @@ import { getMigrationsTxs } from '../staking/2.0/migrations'; import { MIGRATION_SEPSP2_100_PERCENT_KEY } from '../staking/2.0/utils'; import { grp2ConfigByChain } from '../../../src/lib/gas-refund/config'; import { assert } from 'ts-essentials'; +import { DuneTransaction } from '../../../src/models/DuneTransaction'; +import { Op } from 'sequelize'; import { fetchTxGasUsed } from '../../../src/lib/fetch-tx-gas-used'; import { ExtendedCovalentGasRefundTransaction } from '../../../src/types-from-scripts'; @@ -89,7 +95,19 @@ export const getContractAddresses = ({ ); } - return contractAddressesByChain[chainId] ?? [AUGUSTUS_V5_ADDRESS]; + const result = contractAddressesByChain[chainId] ?? [AUGUSTUS_V5_ADDRESS]; + + const withoutDelta = [ + ...result, + '0x00000000FdAC7708D0D360BDDc1bc7d097F47439'.toLowerCase(), // augustus 6.0 + '0x000db803a70511e09da650d4c0506d0000100000'.toLowerCase(), // augustus 6.1 + ]; + + return chainId == 1 + ? withoutDelta + : withoutDelta.concat( + '0x6a000f20005980200259b80c5102003040001068'.toLowerCase(), + ); // augustus 6.2 -> with delta }; export const getAllTXs = async ({ @@ -101,14 +119,27 @@ export const getAllTXs = async ({ contractAddress, }: GetAllTXsInput): Promise => { // fetch swaps and contract (staking pools, safety module) txs - if (contractAddress === AUGUSTUS_V5_ADDRESS && chainId !== CHAIN_ID_OPTIMISM) - return getSwapTXs({ - epoch, - chainId, - startTimestamp, - endTimestamp, - epochEndTimestamp, - }); + if ( + contractAddress === AUGUSTUS_V5_ADDRESS && + chainId !== CHAIN_ID_OPTIMISM + ) { + const swapTxs = ( + await DuneTransaction.findAll({ + where: { + chainId, + to: contractAddress, + block_timestamp: { + [Op.gt]: startTimestamp, + [Op.lte]: endTimestamp, + }, + success: true, + }, + }) + ) + .map(duneToCovalentLike) + .map(constructCovalentAddressToTransaction(contractAddress, chainId)); + return swapTxs; + } if (contractAddress === MIGRATION_SEPSP2_100_PERCENT_KEY) { return getMigrationsTxs({ @@ -119,13 +150,22 @@ export const getAllTXs = async ({ }); } - return getTransactionForContract({ - epoch, - chainId, - startTimestamp, - endTimestamp, - contractAddress, - }); + const filteredTxs = ( + await DuneTransaction.findAll({ + where: { + chainId, + to: contractAddress, + block_timestamp: { + [Op.gt]: startTimestamp, + [Op.lte]: endTimestamp, + }, + }, + }) + ) + .map(duneToCovalentLike) + .map(constructCovalentAddressToTransaction(contractAddress, chainId)); + + return filteredTxs; }; /** @@ -140,7 +180,9 @@ type GetSwapTXsInput = { endTimestamp: number; epochEndTimestamp: number; }; -export const getSwapTXs = async ({ + +// not used if replaced by dune +const getSwapTXs = async ({ epoch, chainId, startTimestamp, diff --git a/scripts/gas-refund-program/transactions-indexing/txs-covalent.ts b/scripts/gas-refund-program/transactions-indexing/txs-covalent.ts index a345340f..690801c1 100644 --- a/scripts/gas-refund-program/transactions-indexing/txs-covalent.ts +++ b/scripts/gas-refund-program/transactions-indexing/txs-covalent.ts @@ -5,11 +5,14 @@ import { } from '../../../src/lib/utils/covalent'; import { covalentClient } from '../../../src/lib/utils/data-providers-clients'; import { CHAIN_ID_OPTIMISM } from '../../../src/lib/constants'; +import { DuneTransaction } from '../../../src/models/DuneTransaction'; import { CovalentTransaction, CovalentAPI, ExtendedCovalentGasRefundTransaction, } from '../../../src/types-from-scripts'; +import { GasRefundTransaction } from '../../../src/models/GasRefundTransaction'; +import { parse } from 'path'; interface GetContractTXsByNetworkInput { chainId: number; @@ -168,3 +171,60 @@ export const covalentGetTXsForContractV3 = async ({ return filteredTxs; }; + +export function duneToCovalentLike( + dbTx: DuneTransaction, +): CovalentTransactionV3 { + return { + to_address: dbTx.to, + tx_hash: dbTx.hash, + from_address: dbTx.from, + gas_price: parseInt(dbTx.gas_price), + gas_spent: dbTx.gas_used, + block_height: dbTx.block_number, + block_signed_at: new Date(dbTx.block_timestamp * 1000) + .toISOString() + .replace(/\.\d+Z$/, 'Z'), + fees_paid: ( + BigInt(dbTx.gas_used) * BigInt(dbTx.gas_price) + + BigInt(dbTx.l1_fee || 0) + ).toString(), + successful: dbTx.success, + }; +} + +export const constructCovalentAddressToTransaction = ( + contract: string, + chainId: number, +) => { + return ( + txCov: CovalentTransactionV3, + ): ExtendedCovalentGasRefundTransaction => { + const { + tx_hash: txHash, + from_address: txOrigin, + gas_price: _txGasPrice, + gas_spent: txGasUsed, + block_height: blockNumber, + block_signed_at: blockTimestamp, + fees_paid: feesPaidInChainCurrency, + } = txCov; + + const timestamp = (new Date(blockTimestamp).getTime() / 1000).toString(); // convert time to unixtime (seconds) + + const txGasPrice = + chainId !== CHAIN_ID_OPTIMISM + ? _txGasPrice.toString() + : (BigInt(feesPaidInChainCurrency) / BigInt(txGasUsed)).toString(); // virtually scaling gasPrice up for optimism to take into account for L1 tx fees submission (dirty fix, shouldn't cause too much troubles) + + return { + txHash, + txOrigin, + txGasPrice, + txGasUsed: txGasUsed.toString(), + blockNumber: blockNumber.toString(), + timestamp: timestamp, + contract, + }; + }; +}; diff --git a/scripts/gas-refund-program/transactions-validation/validateTransactions.ts b/scripts/gas-refund-program/transactions-validation/validateTransactions.ts index 3b9477c5..8f430acf 100644 --- a/scripts/gas-refund-program/transactions-validation/validateTransactions.ts +++ b/scripts/gas-refund-program/transactions-validation/validateTransactions.ts @@ -222,16 +222,34 @@ export async function validateTransactions() { let cappedRefundedAmountPSP: BigNumber | undefined; let cappedRefundedAmountUSD: BigNumber | undefined; + // (guardian.isMaxYearlyPSPGlobalBudgetSpent() || + // guardian.hasSpentYearlyUSDBudget(address) || + // (tx.epoch >= GasRefundBudgetLimitEpochBasedStartEpoch && + // guardian.hasSpentUSDBudgetForEpoch(address, tx.epoch)) || + // (tx.epoch >= GasRefundDeduplicationStartEpoch && + // uniqTxHashesForEpoch.has(hashKey(tx))) || // prevent double spending overall + // migrationsTxsHashesSet.has(tx.hash)) // avoid double spending for twin migration txs (with contract set to actual contract address). Order of txs matters + const rejectReasons = { + yearlyPSP: guardian.isMaxYearlyPSPGlobalBudgetSpent(), + yearlyUSD: guardian.hasSpentYearlyUSDBudget(address), + epochUSD: + tx.epoch >= GasRefundBudgetLimitEpochBasedStartEpoch && + guardian.hasSpentUSDBudgetForEpoch(address, tx.epoch), + deduplication: + tx.epoch >= GasRefundDeduplicationStartEpoch && + uniqTxHashesForEpoch.has(hashKey(tx)), + migration: migrationsTxsHashesSet.has(tx.hash), + }; + + const truthyRectionReasons = Object.entries(rejectReasons).filter( + ([_, value]) => value, + ); + if ( !isMigrationToV2Tx && // always refund migration txs (100%) - (guardian.isMaxYearlyPSPGlobalBudgetSpent() || - guardian.hasSpentYearlyUSDBudget(address) || - (tx.epoch >= GasRefundBudgetLimitEpochBasedStartEpoch && - guardian.hasSpentUSDBudgetForEpoch(address, tx.epoch)) || - (tx.epoch >= GasRefundDeduplicationStartEpoch && - uniqTxHashesForEpoch.has(hashKey(tx))) || // prevent double spending overall - migrationsTxsHashesSet.has(tx.hash)) // avoid double spending for twin migration txs (with contract set to actual contract address). Order of txs matters + truthyRectionReasons.length > 0 ) { + debugger; newStatus = TransactionStatus.REJECTED; } else { newStatus = TransactionStatus.VALIDATED; diff --git a/src/database.ts b/src/database.ts index dfc50d1c..d45ef63e 100644 --- a/src/database.ts +++ b/src/database.ts @@ -51,7 +51,14 @@ export class Database { const connectionString = connectionStringParts.join('/'); this.sequelize = new Sequelize(connectionString, { - logging: IS_DEV ? msg => logger.debug(msg) : undefined, + logging: IS_DEV + ? ( + msg, // avoid huge insert queries stuffing stdout + ) => + logger.debug( + msg.includes('INSERT INTO') ? msg.substring(0, 300) : msg, + ) + : undefined, models: [__dirname + '/models'], // needed locally to connect to docker db ...(IS_DEV && { diff --git a/src/lib/provider.ts b/src/lib/provider.ts index 7da444fe..1be4563f 100644 --- a/src/lib/provider.ts +++ b/src/lib/provider.ts @@ -1,4 +1,7 @@ -import { JsonRpcProvider } from '@ethersproject/providers'; +import { + JsonRpcBatchProvider, + JsonRpcProvider, +} from '@ethersproject/providers'; import { Web3Provider } from './constants'; import { retryDecorator } from 'ts-retry-promise'; diff --git a/src/lib/utils/aura-rewards.ts b/src/lib/utils/aura-rewards.ts index f5f0b417..03e11b78 100644 --- a/src/lib/utils/aura-rewards.ts +++ b/src/lib/utils/aura-rewards.ts @@ -35,6 +35,7 @@ const config: Record = { 48: '0', 49: '0', 50: '1309139600000000000000000', + 51: '0', }; // debugger; const AURA_REWARDS_START_EPOCH_OLD_STYLE = Math.min( @@ -320,6 +321,10 @@ async function computeUserRewardWei( export async function composeAuraRewards( epochOldStyle: number, ): Promise { + if (config[epochOldStyle] === '0') { + // not distributing aura rewards this time? return empty array + return []; + } const { sePSP2BalancesByUserByChain, totalSupplySePSP2 } = await fetchPastEpochData(epochOldStyle - GasRefundV2EpochFlip); diff --git a/src/lib/utils/covalent.ts b/src/lib/utils/covalent.ts index dd4fbff6..444f8212 100644 --- a/src/lib/utils/covalent.ts +++ b/src/lib/utils/covalent.ts @@ -147,8 +147,9 @@ export interface CovalentTransactionV3 { gas_price: number; block_height: number; block_signed_at: string; - gas_spent: string; + gas_spent: number; fees_paid: string; + successful: boolean; } interface MinBulkTimeBucketTxsResponse { diff --git a/src/lib/utils/http-client.ts b/src/lib/utils/http-client.ts index ed3fff04..63000e68 100644 --- a/src/lib/utils/http-client.ts +++ b/src/lib/utils/http-client.ts @@ -82,6 +82,7 @@ export const constructHttpClient = ( ...(options?.retryOptions || {}), }); + if (process.env.NODE_ENV === 'development') axiosCurlirize(client); if (shouldCurlize) axiosCurlirize(client); return client; diff --git a/src/models/DuneTransaction.ts b/src/models/DuneTransaction.ts new file mode 100644 index 00000000..4cce7995 --- /dev/null +++ b/src/models/DuneTransaction.ts @@ -0,0 +1,72 @@ +import { + Table, + Model, + Column, + PrimaryKey, + DataType, + AutoIncrement, +} from 'sequelize-typescript'; + +import { + DataType_ADDRESS, + DataType_KECCAK256_HASHED_VALUE, +} from '../lib/sql-data-types'; + +export type DuneRow = Partial<{ + to: string; + hash: string; // "0x2d1ee3a53c73556a730084043664ed85b4786fd6de333c1f8c81b66509360e68", + from: string; + gas_price: string; // "100000058", + gas_used: number; + block_number: number; + l1_fee: string; + success: boolean; + + // network: string; + chainId: number; + contract: string; + + block_time: string; + block_timestamp: number; // decorated +}>; + +@Table +export class DuneTransaction extends Model { + @PrimaryKey + @AutoIncrement + @Column(DataType.INTEGER) + id: number; + + @Column(DataType.INTEGER) + chainId: number; + + @Column(DataType_ADDRESS) + to: string; + + @Column(DataType_KECCAK256_HASHED_VALUE) + hash: string; + + @Column(DataType_ADDRESS) + from: string; + + @Column(DataType_ADDRESS) + contract: string; + + @Column(DataType.INTEGER) + block_number: number; + + @Column(DataType.DECIMAL) + gas_price: string; + + @Column(DataType.INTEGER) + gas_used: number; + + @Column(DataType.INTEGER) + block_timestamp: number; // decorated + + @Column(DataType.DECIMAL) + l1_fee: string; + + @Column(DataType.BOOLEAN) + success: boolean; +}