From 1fcba401acdf446ee35331cd4ba935ca4f8cdf14 Mon Sep 17 00:00:00 2001 From: phamphong9981 Date: Mon, 5 Aug 2024 15:58:31 +0700 Subject: [PATCH 1/2] feat: erc20 reindex too big contract --- ci/config.json.ci | 5 +- config.json | 5 +- src/services/evm/erc20.service.ts | 13 ++-- src/services/evm/erc20_handler.ts | 31 +++++++- src/services/evm/erc20_reindex.ts | 122 ++++++++++++++++++++---------- 5 files changed, 124 insertions(+), 52 deletions(-) diff --git a/ci/config.json.ci b/ci/config.json.ci index a0239a5f2..b51e73d41 100644 --- a/ci/config.json.ci +++ b/ci/config.json.ci @@ -404,7 +404,10 @@ "blocksPerCall": 100, "millisecondRepeatJob": 5000, "chunkSizeInsert": 1000, - "wrapExtensionContract": ["0xe974cc14c93fc6077b0d65f98832b846c5454a0b"] + "wrapExtensionContract": ["0xe974cc14c93fc6077b0d65f98832b846c5454a0b"], + "reindex": { + "limitRecordGet": 2000 + } }, "erc721": { "key": "erc721", diff --git a/config.json b/config.json index 0318fa189..2f27db9da 100644 --- a/config.json +++ b/config.json @@ -402,7 +402,10 @@ "blocksPerCall": 100, "millisecondRepeatJob": 2000, "chunkSizeInsert": 1000, - "wrapExtensionContract": ["0xe974cc14c93fc6077b0d65f98832b846c5454a0b"] + "wrapExtensionContract": ["0xe974cc14c93fc6077b0d65f98832b846c5454a0b"], + "reindex": { + "limitRecordGet": 2000 + } }, "erc721": { "key": "erc721", diff --git a/src/services/evm/erc20.service.ts b/src/services/evm/erc20.service.ts index a50edbd8e..a6e490d51 100644 --- a/src/services/evm/erc20.service.ts +++ b/src/services/evm/erc20.service.ts @@ -90,13 +90,12 @@ export default class Erc20Service extends BullableService { ], config.erc20.key ); - const erc20Activities: Erc20Activity[] = - await Erc20Handler.buildErc20Activities( - startBlock, - endBlock, - trx, - this.logger - ); + const { erc20Activities } = await Erc20Handler.buildErc20Activities( + startBlock, + endBlock, + trx, + this.logger + ); await this.handleMissingErc20Contract(erc20Activities, trx); if (erc20Activities.length > 0) { this.logger.info( diff --git a/src/services/evm/erc20_handler.ts b/src/services/evm/erc20_handler.ts index 4edc85117..e6fa76a44 100644 --- a/src/services/evm/erc20_handler.ts +++ b/src/services/evm/erc20_handler.ts @@ -170,7 +170,12 @@ export class Erc20Handler { endBlock: number, trx: Knex.Transaction, logger: Moleculer.LoggerInstance, - addresses?: string[] + addresses?: string[], + page?: { + prevEvmEventId?: number; + limitRecordGet: number; + prevCosmosEventId?: number; + } ) { const erc20Activities: Erc20Activity[] = []; const erc20Events = await EvmEvent.query() @@ -185,6 +190,11 @@ export class Erc20Handler { if (addresses) { builder.whereIn('evm_event.address', addresses); } + if (page && page.prevEvmEventId !== undefined) { + builder + .andWhere('evm_event.id', '>', page.prevEvmEventId) + .limit(page.limitRecordGet); + } }) .where('evm_event.block_height', '>', startBlock) .andWhere('evm_event.block_height', '<=', endBlock) @@ -213,6 +223,11 @@ export class Erc20Handler { .where('attributes.key', EventAttribute.ATTRIBUTE_KEY.ERC20_TOKEN) .whereIn(knex.raw('lower("value")'), addresses); } + if (page && page.prevCosmosEventId !== undefined) { + builder + .andWhere('event.id', '>', page.prevCosmosEventId) + .limit(page.limitRecordGet); + } }) .withGraphFetched('[transaction, attributes]') .orderBy('event.id', 'asc'); @@ -244,14 +259,19 @@ export class Erc20Handler { erc20Activities.push(activity); } }); - return _.sortBy(erc20Activities, 'cosmos_tx_id'); + return { + erc20Activities: _.sortBy(erc20Activities, 'cosmos_tx_id'), + prevEvmEventId: erc20Events[erc20Events.length - 1]?.id, + prevCosmosEventId: erc20CosmosEvents[erc20CosmosEvents.length - 1]?.id, + }; } static async getErc20Activities( startBlock: number, endBlock: number, trx?: Knex.Transaction, - addresses?: string[] + addresses?: string[], + page?: { prevId: number; limitRecordGet: number } ): Promise { return Erc20Activity.query() .modify((builder) => { @@ -261,6 +281,11 @@ export class Erc20Handler { if (trx) { builder.transacting(trx); } + if (page) { + builder + .andWhere('erc20_activity.id', '>', page.prevId) + .limit(page.limitRecordGet); + } }) .leftJoin( 'account as from_account', diff --git a/src/services/evm/erc20_reindex.ts b/src/services/evm/erc20_reindex.ts index 4838aeca8..77f00fedc 100644 --- a/src/services/evm/erc20_reindex.ts +++ b/src/services/evm/erc20_reindex.ts @@ -1,3 +1,4 @@ +/* eslint-disable no-await-in-loop */ import Moleculer from 'moleculer'; import { getContract, PublicClient } from 'viem'; import config from '../../../config.json' assert { type: 'json' }; @@ -74,49 +75,90 @@ export class Erc20Reindexer { }) ) .transacting(trx); - const erc20Activities = await Erc20Handler.buildErc20Activities( - 0, - Number(blockHeight), - trx, - this.logger, - [address] - ); - if (erc20Activities.length > 0) { - await knex - .batchInsert( - 'erc20_activity', - erc20Activities, - config.erc20.chunkSizeInsert - ) - .transacting(trx); + const {limitRecordGet} = config.erc20.reindex; + let prevEvmEventId = 0; + let prevCosmosEventId = '0'; + let numActivities = 0; + while (true) { + const resultBuildErc20Activities = + await Erc20Handler.buildErc20Activities( + 0, + Number(blockHeight), + trx, + this.logger, + [address], + { + prevEvmEventId, + limitRecordGet, + } + ); + const {erc20Activities} = resultBuildErc20Activities; + if (erc20Activities.length > 0) { + const resultInsert: any = await knex + .batchInsert( + 'erc20_activity', + erc20Activities, + config.erc20.chunkSizeInsert + ) + .transacting(trx); + numActivities += resultInsert[0].rowCount; + } + prevEvmEventId = resultBuildErc20Activities.prevEvmEventId; + prevCosmosEventId = resultBuildErc20Activities.prevCosmosEventId; + if (prevEvmEventId === undefined && prevCosmosEventId === undefined) { + break; + } } - const erc20ActivitiesInDb = await Erc20Handler.getErc20Activities( - 0, - Number(blockHeight), - trx, - [address] - ); - // get missing Account - const missingAccountsAddress = Array.from( - new Set( - [ - ...erc20ActivitiesInDb - .filter((e) => !e.from_account_id) - .map((e) => e.from), - ...erc20ActivitiesInDb - .filter((e) => !e.to_account_id) - .map((e) => e.to), - ].map((e) => - convertEthAddressToBech32Address(config.networkPrefixAddress, e) - ) as string[] - ) - ); - if (missingAccountsAddress.length > 0) { - throw new Error( - `Missing accounts ${missingAccountsAddress}. You should reindex them` + this.logger.info(`Reindex erc20 activities ${address} done.`); + let prevId = 0; + let numChunk = 1; + while (true) { + const erc20ActivitiesInDb = await Erc20Handler.getErc20Activities( + 0, + Number(blockHeight), + trx, + [address], + { + prevId, + limitRecordGet, + } + ); + // get missing Account + const missingAccountsAddress = Array.from( + new Set( + [ + ...erc20ActivitiesInDb + .filter((e) => !e.from_account_id) + .map((e) => e.from), + ...erc20ActivitiesInDb + .filter((e) => !e.to_account_id) + .map((e) => e.to), + ].map((e) => + convertEthAddressToBech32Address(config.networkPrefixAddress, e) + ) as string[] + ) ); + if (missingAccountsAddress.length > 0) { + throw new Error( + `Missing accounts ${missingAccountsAddress}. You should reindex them` + ); + } + if (erc20ActivitiesInDb.length > 0) { + await Erc20Handler.updateErc20AccountsBalance( + erc20ActivitiesInDb, + trx + ); + prevId = erc20ActivitiesInDb[erc20ActivitiesInDb.length - 1].id; + this.logger.info( + `Reindex erc20 contract ${address}: Chunk ${numChunk}/${Math.floor( + numActivities / limitRecordGet + 1 + )} done` + ); + numChunk += 1; + } else { + break; + } } - await Erc20Handler.updateErc20AccountsBalance(erc20ActivitiesInDb, trx); }); } } From e3740cade8d8282848484248a5c3ab8a2b06e951 Mon Sep 17 00:00:00 2001 From: phamphong9981 Date: Tue, 6 Aug 2024 10:22:19 +0700 Subject: [PATCH 2/2] fix: test --- test/unit/services/evm/erc20_handler.spec.ts | 26 ++++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test/unit/services/evm/erc20_handler.spec.ts b/test/unit/services/evm/erc20_handler.spec.ts index ba2e248f3..c17fdff64 100644 --- a/test/unit/services/evm/erc20_handler.spec.ts +++ b/test/unit/services/evm/erc20_handler.spec.ts @@ -187,15 +187,15 @@ export default class Erc20HandlerTest { ]; await EvmEvent.query().insert(evmEvents); await knex.transaction(async (trx) => { - const erc20Activitites = await Erc20Handler.buildErc20Activities( + const { erc20Activities } = await Erc20Handler.buildErc20Activities( blockHeight - 1, blockHeight, trx, this.broker.logger ); - expect(erc20Activitites.length).toEqual(evmEvents.length - 1); + expect(erc20Activities.length).toEqual(evmEvents.length - 1); // test build transfer activity - const transferActivity = erc20Activitites[0]; + const transferActivity = erc20Activities[0]; expect(transferActivity).toMatchObject({ action: ERC20_ACTION.TRANSFER, erc20_contract_address: erc20Contract.address, @@ -204,7 +204,7 @@ export default class Erc20HandlerTest { amount, }); // test build approve activity - const approvalActivity = erc20Activitites[1]; + const approvalActivity = erc20Activities[1]; expect(approvalActivity).toMatchObject({ action: ERC20_ACTION.APPROVAL, erc20_contract_address: erc20Contract.address, @@ -269,15 +269,15 @@ export default class Erc20HandlerTest { ]; await EvmEvent.query().insert(evmEvents); await knex.transaction(async (trx) => { - const erc20Activitites = await Erc20Handler.buildErc20Activities( + const { erc20Activities } = await Erc20Handler.buildErc20Activities( blockHeight - 1, blockHeight, trx, this.broker.logger ); - expect(erc20Activitites.length).toEqual(evmEvents.length); + expect(erc20Activities.length).toEqual(evmEvents.length); // test build deposit activity - const depositActivity = erc20Activitites[0]; + const depositActivity = erc20Activities[0]; expect(depositActivity).toMatchObject({ action: ERC20_ACTION.DEPOSIT, erc20_contract_address: erc20WrapContract.address, @@ -286,7 +286,7 @@ export default class Erc20HandlerTest { amount, }); // test build withdrawal activity - const withdrawalActivity = erc20Activitites[1]; + const withdrawalActivity = erc20Activities[1]; expect(withdrawalActivity).toMatchObject({ action: ERC20_ACTION.WITHDRAWAL, erc20_contract_address: erc20WrapContract.address, @@ -470,14 +470,14 @@ export default class Erc20HandlerTest { ]; await EvmEvent.query().insert(evmEvents); await knex.transaction(async (trx) => { - const erc20Activitites = await Erc20Handler.buildErc20Activities( + const { erc20Activities } = await Erc20Handler.buildErc20Activities( blockHeight - 2, blockHeight + 1, trx, this.broker.logger ); // test convert coin activity - const convertCoinActivity = erc20Activitites[2]; + const convertCoinActivity = erc20Activities[2]; expect(convertCoinActivity).toMatchObject({ from: ZERO_ADDRESS, to, @@ -487,7 +487,7 @@ export default class Erc20HandlerTest { cosmos_event_id: transaction.events[0].id, }); // test convert erc20 activity - const convertErc20Activity = erc20Activitites[1]; + const convertErc20Activity = erc20Activities[1]; expect(convertErc20Activity).toMatchObject({ from, to: ZERO_ADDRESS, @@ -497,7 +497,7 @@ export default class Erc20HandlerTest { cosmos_event_id: transaction.events[1].id, }); // test sort order - const transferActivity1 = erc20Activitites[0]; + const transferActivity1 = erc20Activities[0]; expect(transferActivity1).toMatchObject({ action: ERC20_ACTION.TRANSFER, erc20_contract_address: erc20Contract.address, @@ -507,7 +507,7 @@ export default class Erc20HandlerTest { evm_tx_id: evmEvents[0].evm_tx_id, cosmos_tx_id: evmEvents[0].tx_id, }); - const transferActivity2 = erc20Activitites[3]; + const transferActivity2 = erc20Activities[3]; expect(transferActivity2).toMatchObject({ action: ERC20_ACTION.TRANSFER, erc20_contract_address: erc20Contract.address,