From 9c7fb1db2b9ac5d92563c546b345a04a159e58c8 Mon Sep 17 00:00:00 2001 From: tmcgroul Date: Wed, 25 Sep 2024 04:58:25 +0700 Subject: [PATCH] adjust tron-usdt test project --- common/config/rush/pnpm-lock.yaml | 14 +- rush.json | 6 + test/tron-usdt/package.json | 1 + test/tron-usdt/src/main.ts | 47 +++- tron/tron-data/src/data-source.ts | 2 +- tron/tron-normalization/src/data.ts | 14 +- tron/tron-normalization/src/mapping.ts | 27 +- tron/tron-objects/package.json | 29 +++ tron/tron-objects/src/augment.ts | 11 + tron/tron-objects/src/index.ts | 2 + tron/tron-objects/src/items.ts | 179 ++++++++++++++ tron/tron-objects/src/relations.ts | 32 +++ tron/tron-objects/src/types.ts | 37 +++ tron/tron-objects/src/util.ts | 15 ++ tron/tron-objects/tsconfig.json | 21 ++ tron/tron-stream/src/ds-archive.ts | 232 ------------------ tron/tron-stream/src/http/filter.ts | 187 ++++++++++++++ .../src/{http.ts => http/source.ts} | 22 +- tron/tron-stream/src/index.ts | 3 +- tron/tron-stream/src/mapping.ts | 224 ----------------- tron/tron-stream/src/source.ts | 7 +- 21 files changed, 634 insertions(+), 478 deletions(-) create mode 100644 tron/tron-objects/package.json create mode 100644 tron/tron-objects/src/augment.ts create mode 100644 tron/tron-objects/src/index.ts create mode 100644 tron/tron-objects/src/items.ts create mode 100644 tron/tron-objects/src/relations.ts create mode 100644 tron/tron-objects/src/types.ts create mode 100644 tron/tron-objects/src/util.ts create mode 100644 tron/tron-objects/tsconfig.json delete mode 100644 tron/tron-stream/src/ds-archive.ts create mode 100644 tron/tron-stream/src/http/filter.ts rename tron/tron-stream/src/{http.ts => http/source.ts} (55%) delete mode 100644 tron/tron-stream/src/mapping.ts diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 868e9be28..bb9909d20 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -227,6 +227,9 @@ dependencies: '@rush-temp/tron-normalization': specifier: file:./projects/tron-normalization.tgz version: file:projects/tron-normalization.tgz + '@rush-temp/tron-objects': + specifier: file:./projects/tron-objects.tgz + version: file:projects/tron-objects.tgz '@rush-temp/tron-stream': specifier: file:./projects/tron-stream.tgz version: file:projects/tron-stream.tgz @@ -7906,6 +7909,15 @@ packages: typescript: 5.3.2 dev: false + file:projects/tron-objects.tgz: + resolution: {integrity: sha512-6rqyj1YmfMIli3yV9cUb7q3RZ+2HlJE3qNFCqeFDCONYFeEL1ZbnENgpVuw4PIkiw3WXSP/ZKQ5O3BNYBapCmQ==, tarball: file:projects/tron-objects.tgz} + name: '@rush-temp/tron-objects' + version: 0.0.0 + dependencies: + '@types/node': 18.19.0 + typescript: 5.3.2 + dev: false + file:projects/tron-stream.tgz: resolution: {integrity: sha512-aOQDmcSsbMo84b3AUzjqjumVq7WLmfuP/+ouWThViyOxZo3jnMn25kLwhgts7I1iUjE85aQsCpwLe/NRggAIxg==, tarball: file:projects/tron-stream.tgz} name: '@rush-temp/tron-stream' @@ -7916,7 +7928,7 @@ packages: dev: false file:projects/tron-usdt.tgz(supports-color@8.1.1)(ts-node@10.9.2): - resolution: {integrity: sha512-HFWa/F00GcWBxAr96yjNylEsws0jvqvNzfLy4En8BtjhmZKZAUiD0IMwOaP8ZUz6PkMR/dgdOuvcELKctGj+ZA==, tarball: file:projects/tron-usdt.tgz} + resolution: {integrity: sha512-Dz6cLv5n8AvqP9CXDgC02fncVXbi/A74Gp2wap+ORNqntP0pHSBMtlkfxKDsOSXEcoIS4GlFRe+AF/BaHoWlVw==, tarball: file:projects/tron-usdt.tgz} id: file:projects/tron-usdt.tgz name: '@rush-temp/tron-usdt' version: 0.0.0 diff --git a/rush.json b/rush.json index 106c690aa..5b0927715 100644 --- a/rush.json +++ b/rush.json @@ -645,6 +645,12 @@ "shouldPublish": true, "versionPolicyName": "npm" }, + { + "packageName": "@subsquid/tron-objects", + "projectFolder": "tron/tron-objects", + "shouldPublish": true, + "versionPolicyName": "npm" + }, { "packageName": "@subsquid/tron-stream", "projectFolder": "tron/tron-stream", diff --git a/test/tron-usdt/package.json b/test/tron-usdt/package.json index c017006e8..007ce0fba 100644 --- a/test/tron-usdt/package.json +++ b/test/tron-usdt/package.json @@ -9,6 +9,7 @@ "@subsquid/batch-processor": "^0.0.0", "@subsquid/graphql-server": "^4.7.0", "@subsquid/tron-stream": "^0.0.0", + "@subsquid/tron-objects": "^0.0.0", "@subsquid/typeorm-migration": "^1.3.0", "@subsquid/typeorm-store": "^1.5.1", "dotenv": "^16.3.1", diff --git a/test/tron-usdt/src/main.ts b/test/tron-usdt/src/main.ts index b90418721..e288bcf77 100644 --- a/test/tron-usdt/src/main.ts +++ b/test/tron-usdt/src/main.ts @@ -1,15 +1,27 @@ import {run} from '@subsquid/batch-processor' -import {DataSourceBuilder, assertNotNull} from '@subsquid/tron-stream' +import {augmentBlock} from '@subsquid/tron-objects' +import {DataSourceBuilder} from '@subsquid/tron-stream' import {TypeormDatabase} from '@subsquid/typeorm-store' import * as erc20 from './abi/erc20' import {Transfer} from './model' +const CONTRACT = 'a614f803b6fd780986a42c78ec9c7f77e6ded13c' +const TOPIC0 = 'ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' + + const dataSource = new DataSourceBuilder() - .setHttpApi({ - url: assertNotNull(process.env.TRON_HTTP_API) + .setGateway('https://v2.archive.subsquid.io/network/tron-mainnet') + .setBlockRange({from: 11322942, to: 11323358}) + .addLog({ + where: { + address: [CONTRACT], + topic0: [TOPIC0] + }, + include: { + transaction: true + } }) - .includeAllBlocks() .build() @@ -17,7 +29,30 @@ const database = new TypeormDatabase() run(dataSource, database, async ctx => { - for (let block of ctx.blocks) { - console.log(block) + let transfers: Transfer[] = [] + + let blocks = ctx.blocks.map(augmentBlock) + + for (let block of blocks) { + for (let log of block.logs) { + if (log.address == CONTRACT && log.topics[0] === TOPIC0) { + log.topics = log.topics.map(t => '0x' + t) + log.data = '0x' + log.data + let {from, to, value} = erc20.events.Transfer.decode(log) + let tx = log.getTransaction() + + transfers.push(new Transfer({ + id: log.id, + blockNumber: block.header.height, + timestamp: new Date(block.header.timestamp), + tx: tx.hash, + from, + to, + amount: value + })) + } + } } + + await ctx.store.insert(transfers) }) diff --git a/tron/tron-data/src/data-source.ts b/tron/tron-data/src/data-source.ts index 1e448fa4b..13db79b64 100644 --- a/tron/tron-data/src/data-source.ts +++ b/tron/tron-data/src/data-source.ts @@ -37,7 +37,7 @@ export class HttpDataSource { this.headPollInterval = options.headPollInterval ?? 1000 this.strideSize = options.strideSize || 10 this.strideConcurrency = options.strideConcurrency || 2 - this.finalityConfirmation = 0 + this.finalityConfirmation = 20 } getBlockHeader(height: number) { diff --git a/tron/tron-normalization/src/data.ts b/tron/tron-normalization/src/data.ts index b173fd0fd..105e87cd8 100644 --- a/tron/tron-normalization/src/data.ts +++ b/tron/tron-normalization/src/data.ts @@ -54,6 +54,14 @@ export interface Transaction { netFee?: number originEnergyUsage?: number energyPenaltyTotal?: number + _transferContractOwner?: string + _transferContractTo?: string + _transferAssetContractOwner?: string + _transferAssetContractTo?: string + _transferAssetContractAsset?: string + _triggerSmartContractOwner?: string + _triggerSmartContractContract?: string + _triggerSmartContractSighash?: string } @@ -78,7 +86,7 @@ export interface InternalTransaction { export interface Block { header: BlockHeader, - logs?: Log[] - transactions?: Transaction[] - internalTransactions?: InternalTransaction[] + logs: Log[] + transactions: Transaction[] + internalTransactions: InternalTransaction[] } diff --git a/tron/tron-normalization/src/mapping.ts b/tron/tron-normalization/src/mapping.ts index 1a1f7ccd1..4be95fcf6 100644 --- a/tron/tron-normalization/src/mapping.ts +++ b/tron/tron-normalization/src/mapping.ts @@ -1,8 +1,18 @@ import * as raw from '@subsquid/tron-data' +import {assertNotNull} from '@subsquid/util-internal' import assert from 'assert' import {Block, BlockHeader, InternalTransaction, Log, Transaction} from './data' +function toSighash(val?: string) { + if (val && val.length >= 8) { + return val.slice(0, 8) + } else { + return undefined + } +} + + function mapBlockHeader(src: raw.Block): BlockHeader { return { hash: src.blockID, @@ -21,7 +31,7 @@ function mapTransaction(src: raw.Transaction, transactionIndex: number, info?: r assert(src.raw_data.contract.length == 1) if (info) assert(info.contractResult.length == 1) let contract = src.raw_data.contract[0] - return { + let tx: Transaction = { hash: src.txID, transactionIndex, ret: src.ret, @@ -52,6 +62,21 @@ function mapTransaction(src: raw.Transaction, transactionIndex: number, info?: r originEnergyUsage: info?.receipt.origin_energy_usage, energyPenaltyTotal: info?.receipt.energy_penalty_total, } + + if (tx.type == 'TransferContract') { + tx._transferContractTo = assertNotNull(tx.parameter.value.to_address) + tx._transferContractOwner = assertNotNull(tx.parameter.value.owner_address) + } else if (tx.type == 'TransferAssetContract') { + tx._transferAssetContractAsset = assertNotNull(tx.parameter.value.asset_name) + tx._transferAssetContractOwner = assertNotNull(tx.parameter.value.owner_address) + tx._transferAssetContractTo = assertNotNull(tx.parameter.value.to_address) + } else if (tx.type == 'TriggerSmartContract') { + tx._triggerSmartContractContract = assertNotNull(tx.parameter.value.contract_address) + tx._triggerSmartContractOwner = assertNotNull(tx.parameter.value.owner_address) + tx._triggerSmartContractSighash = toSighash(tx.parameter.value.data) + } + + return tx } diff --git a/tron/tron-objects/package.json b/tron/tron-objects/package.json new file mode 100644 index 000000000..527a29dfa --- /dev/null +++ b/tron/tron-objects/package.json @@ -0,0 +1,29 @@ +{ + "name": "@subsquid/tron-objects", + "version": "0.0.0", + "description": "Augmented Tron data model", + "license": "GPL-3.0-or-later", + "repository": "git@github.com:subsquid/squid.git", + "publishConfig": { + "access": "public" + }, + "main": "lib/index.js", + "files": [ + "lib", + "src" + ], + "scripts": { + "build": "rm -rf lib && tsc" + }, + "dependencies": { + "@subsquid/util-internal": "^3.2.0" + }, + "peerDependencies": { + "@subsquid/tron-stream": "^0.0.0" + }, + "devDependencies": { + "@subsquid/tron-stream": "^0.0.0", + "@types/node": "^18.18.14", + "typescript": "~5.3.2" + } +} diff --git a/tron/tron-objects/src/augment.ts b/tron/tron-objects/src/augment.ts new file mode 100644 index 000000000..0fa48bd66 --- /dev/null +++ b/tron/tron-objects/src/augment.ts @@ -0,0 +1,11 @@ +import * as base from '@subsquid/tron-stream' +import {setUpRelations} from './relations' +import * as types from './types' +import {Block} from './items' + + +export function augmentBlock(src: base.Block): types.Block { + let block = Block.fromPartial(src) + setUpRelations(block) + return block as unknown as types.Block +} diff --git a/tron/tron-objects/src/index.ts b/tron/tron-objects/src/index.ts new file mode 100644 index 000000000..447e04697 --- /dev/null +++ b/tron/tron-objects/src/index.ts @@ -0,0 +1,2 @@ +export * from './types' +export * from './augment' diff --git a/tron/tron-objects/src/items.ts b/tron/tron-objects/src/items.ts new file mode 100644 index 000000000..e79f43dc1 --- /dev/null +++ b/tron/tron-objects/src/items.ts @@ -0,0 +1,179 @@ +import { + PartialBlock, + PartialBlockHeader, + PartialLog, + PartialTransaction, + PartialInternalTransaction +} from '@subsquid/tron-stream/lib/data/data-partial' +import {formatId} from './util' + + +export class Block { + constructor(public header: BlockHeader) {} + + transactions: Transaction[] = [] + logs: Log[] = [] + internalTransactions: InternalTransaction[] = [] + + static fromPartial(src: PartialBlock): Block { + let block = new Block(new BlockHeader(src.header)) + + if (src.transactions) { + block.transactions = src.transactions.map(i => new Transaction(block.header, i)) + } + + if (src.logs) { + block.logs = src.logs.map(i => new Log(block.header, i)) + } + + if (src.internalTransactions) { + block.internalTransactions = src.internalTransactions.map(i => { + return new InternalTransaction(block.header, i) + }) + } + + return block + } +} + + +export class BlockHeader implements PartialBlockHeader { + id: string + height: number + hash: string + + constructor(header: PartialBlockHeader) { + this.id = formatId(header) + this.height = header.height + this.hash = header.hash + Object.assign(this, header) + } +} + + +export class Transaction implements PartialTransaction { + id: string + transactionIndex: number + #block: BlockHeader + #logs?: Log[] + #internalTransactions?: InternalTransaction[] + + constructor(block: BlockHeader, tx: PartialTransaction) { + this.id = formatId(block, tx.transactionIndex) + this.transactionIndex = tx.transactionIndex + this.#block = block + Object.assign(this, tx) + } + + get block(): BlockHeader { + return this.#block + } + + set block(value: BlockHeader) { + this.#block = value + } + + get logs(): Log[] { + if (this.#logs == null) { + this.#logs = [] + } + return this.#logs + } + + set logs(value: Log[]) { + this.#logs = value + } + + get internalTransactions(): InternalTransaction[] { + if (this.#internalTransactions == null) { + this.#internalTransactions = [] + } + return this.#internalTransactions + } + + set internalTransactions(value: InternalTransaction[]) { + this.#internalTransactions = value + } +} + + +export class Log implements PartialLog { + id: string + logIndex: number + transactionIndex: number + #block: BlockHeader + #transaction?: Transaction + + constructor(block: BlockHeader, i: PartialLog) { + this.#block = block + this.id = formatId(block, i.transactionIndex, i.logIndex) + this.logIndex = i.logIndex + this.transactionIndex = i.transactionIndex + Object.assign(this, i) + } + + get block(): BlockHeader { + return this.#block + } + + set block(value: BlockHeader) { + this.#block = value + } + + get transaction(): Transaction | undefined { + return this.#transaction + } + + set transaction(value: Transaction | undefined) { + this.#transaction = value + } + + getTransaction(): Transaction { + if (this.#transaction == null) { + throw new Error(`Transaction is not set on log ${this.id}`) + } else { + return this.#transaction + } + } +} + + +export class InternalTransaction implements PartialInternalTransaction { + id: string + internalTransactionIndex: number + transactionIndex: number + #block: BlockHeader + #transaction?: Transaction + + constructor(block: BlockHeader, src: PartialInternalTransaction) { + this.#block = block + this.id = formatId(block, src.transactionIndex, src.internalTransactionIndex) + this.internalTransactionIndex = src.internalTransactionIndex + this.transactionIndex = src.transactionIndex + Object.assign(this, src) + } + + get block(): BlockHeader { + return this.#block + } + + set block(value: BlockHeader) { + this.#block = value + } + + get transaction(): Transaction | undefined { + return this.#transaction + } + + set transaction(value: Transaction | undefined) { + this.#transaction = value + } + + getTransaction(): Transaction { + if (this.#transaction == null) { + throw new Error(`Transaction is not set on internal transaction ${this.id}`) + } else { + return this.#transaction + } + } +} diff --git a/tron/tron-objects/src/relations.ts b/tron/tron-objects/src/relations.ts new file mode 100644 index 000000000..85b75183f --- /dev/null +++ b/tron/tron-objects/src/relations.ts @@ -0,0 +1,32 @@ +import {maybeLast} from '@subsquid/util-internal' +import {Block, Transaction} from './items' + + +export function setUpRelations(block: Block): void { + block.transactions.sort((a, b) => a.transactionIndex - b.transactionIndex) + block.logs.sort((a, b) => a.transactionIndex - b.transactionIndex || a.logIndex - b.logIndex) + block.internalTransactions.sort((a, b) => { + return a.transactionIndex - b.transactionIndex || a.internalTransactionIndex - b.internalTransactionIndex + }) + + let txs: (Transaction | undefined)[] = new Array((maybeLast(block.transactions)?.transactionIndex ?? -1) + 1) + for (let tx of block.transactions) { + txs[tx.transactionIndex] = tx + } + + for (let log of block.logs) { + let tx = txs[log.transactionIndex] + if (tx) { + log.transaction = tx + tx.logs.push(log) + } + } + + for (let internalTx of block.internalTransactions) { + let tx = txs[internalTx.transactionIndex] + if (tx) { + internalTx.transaction = tx + tx.internalTransactions.push(internalTx) + } + } +} diff --git a/tron/tron-objects/src/types.ts b/tron/tron-objects/src/types.ts new file mode 100644 index 000000000..a17d5f451 --- /dev/null +++ b/tron/tron-objects/src/types.ts @@ -0,0 +1,37 @@ +import {FieldSelection} from '@subsquid/tron-stream' +import * as base from '@subsquid/tron-stream' + + +export type BlockHeader = base.BlockHeader + + +export type Transaction = base.Transaction & { + id: string + block: BlockHeader + logs: Log[] + internalTransactions: InternalTransaction[] +} + + +export type Log = base.Log & { + id: string + block: BlockHeader + transaction?: Transaction + getTransaction(): Transaction +} + + +export type InternalTransaction = base.InternalTransaction & { + id: string + block: BlockHeader + transaction?: Transaction + getTransaction(): Transaction +} + + +export interface Block { + header: BlockHeader + transactions: Transaction[] + logs: Log[] + internalTransactions: InternalTransaction[] +} diff --git a/tron/tron-objects/src/util.ts b/tron/tron-objects/src/util.ts new file mode 100644 index 000000000..f0995b6d6 --- /dev/null +++ b/tron/tron-objects/src/util.ts @@ -0,0 +1,15 @@ +export interface HashAndHeight { + hash: string + height: number +} + + +export function formatId(block: HashAndHeight, ...address: number[]): string { + let no = block.height.toString().padStart(12, '0') + let hash = block.hash.slice(16, 21) + let id = `${no}-${hash}` + for (let index of address) { + id += '-' + index.toString().padStart(6, '0') + } + return id +} diff --git a/tron/tron-objects/tsconfig.json b/tron/tron-objects/tsconfig.json new file mode 100644 index 000000000..deee9f66b --- /dev/null +++ b/tron/tron-objects/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "module": "commonjs", + "target": "es2020", + "outDir": "lib", + "rootDir": "src", + "allowJs": true, + "strict": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "esModuleInterop": true, + "experimentalDecorators": true, + "emitDecoratorMetadata": true, + "skipLibCheck": true + }, + "include": ["src"], + "exclude": [ + "node_modules" + ] +} diff --git a/tron/tron-stream/src/ds-archive.ts b/tron/tron-stream/src/ds-archive.ts deleted file mode 100644 index 0b8e724e1..000000000 --- a/tron/tron-stream/src/ds-archive.ts +++ /dev/null @@ -1,232 +0,0 @@ -// import {annotateSyncError} from '@subsquid/util-internal' -// import {ArchiveClient} from '@subsquid/util-internal-archive-client' -// import {Batch, DataSource} from '@subsquid/util-internal-processor-tools' -// import {RangeRequestList} from '@subsquid/util-internal-range' -// import {archiveIngest} from '@subsquid/util-internal-ingest-tools' -// import {HttpApi} from '@subsquid/tron-data-raw' -// import {ArchiveBlock} from './interfaces/data-partial' -// import {DataRequest} from './interfaces/data-request' -// import {Block, BlockHeader, InternalTransaction, Log, Transaction, setUpItems} from './mapping' - - -// export interface TronArchiveOptions { -// client: ArchiveClient -// httpApi: HttpApi -// } - - -// export class TronArchive implements DataSource { -// private client: ArchiveClient -// private httpApi: HttpApi - -// constructor(options: TronArchiveOptions) { -// this.client = options.client -// this.httpApi = options.httpApi -// } - -// getFinalizedHeight(): Promise { -// return this.client.getHeight() -// } - -// getBlockHash(height: number): Promise { -// return this.httpApi.getBlock(height, false).then(block => block.blockID) -// } - -// async *getFinalizedBlocks(requests: RangeRequestList, stopOnHead?: boolean): AsyncIterable> { -// for await (let {blocks, isHead} of archiveIngest({ -// requests, -// client: this.client, -// stopOnHead -// })) { -// yield { -// blocks: blocks.map(b => this.mapBlock(b)), -// isHead -// } -// } -// } - -// @annotateSyncError((src: ArchiveBlock) => ({blockHeight: src.header.number, blockHash: src.header.hash})) -// private mapBlock(src: ArchiveBlock): Block { -// let block = new Block(new BlockHeader( -// { -// height: src.header.number, -// ...src.header -// } -// )) - -// if (src.transactions) { -// for (let s of src.transactions) { -// let tx = new Transaction(block.header, s.hash) - -// if (s.ret != null) { -// tx.ret = s.ret -// } - -// if (s.signature != null) { -// tx.signature = s.signature -// } - -// if (s.type != null) { -// tx.type = s.type -// } - -// if (s.parameter != null) { -// tx.parameter = s.parameter -// } - -// if (s.permissionId != null) { -// tx.permissionId = s.permissionId -// } - -// if (s.refBlockBytes != null) { -// tx.refBlockBytes = s.refBlockBytes -// } - -// if (s.refBlockHash != null) { -// tx.refBlockHash = s.refBlockHash -// } - -// if (s.feeLimit != null) { -// tx.feeLimit = s.feeLimit -// } - -// if (s.expiration != null) { -// tx.expiration = s.expiration -// } - -// if (s.timestamp != null) { -// tx.timestamp = s.timestamp -// } - -// if (s.rawDataHex != null) { -// tx.rawDataHex = s.rawDataHex -// } - -// if (s.fee != null) { -// tx.fee = s.fee -// } - -// if (s.contractResult != null) { -// tx.contractResult = s.contractResult -// } - -// if (s.contractAddress != null) { -// tx.contractAddress = s.contractAddress -// } - -// if (s.resMessage != null) { -// tx.resMessage = s.resMessage -// } - -// if (s.withdrawAmount != null) { -// tx.withdrawAmount = s.withdrawAmount -// } - -// if (s.unfreezeAmount != null) { -// tx.unfreezeAmount = s.unfreezeAmount -// } - -// if (s.withdrawExpireAmount != null) { -// tx.withdrawExpireAmount = s.withdrawExpireAmount -// } - -// if (s.cancelUnfreezeV2Amount != null) { -// tx.cancelUnfreezeV2Amount = s.cancelUnfreezeV2Amount -// } - -// if (s.result != null) { -// tx.result = s.result -// } - -// if (s.energyFee != null) { -// tx.energyFee = s.energyFee -// } - -// if (s.energyUsage != null) { -// tx.energyUsage = s.energyUsage -// } - -// if (s.energyUsageTotal != null) { -// tx.energyUsageTotal = s.energyUsageTotal -// } - -// if (s.netUsage != null) { -// tx.netUsage = s.netUsage -// } - -// if (s.netFee != null) { -// tx.netFee = s.netFee -// } - -// if (s.originEnergyUsage != null) { -// tx.originEnergyUsage = s.originEnergyUsage -// } - -// if (s.energyPenaltyTotal != null) { -// tx.energyPenaltyTotal = s.energyPenaltyTotal -// } - -// block.transactions.push(tx) -// } -// } - -// if (src.internalTransactions) { -// for (let s of src.internalTransactions) { -// let tx = new InternalTransaction(block.header, s.transactionHash) - -// if (s.hash != null) { -// tx.hash = s.hash -// } - -// if (s.callerAddress != null) { -// tx.callerAddress = s.callerAddress -// } - -// if (s.transferToAddress != null) { -// tx.transferToAddress = s.transferToAddress -// } - -// if (s.callValueInfo != null) { -// tx.callValueInfo = s.callValueInfo -// } - -// if (s.note != null) { -// tx.note = s.note -// } - -// if (s.rejected != null) { -// tx.rejected = s.rejected -// } - -// if (s.extra != null) { -// tx.extra = s.extra -// } - -// block.internalTransactions.push(tx) -// } -// } - -// if (src.logs) { -// for (let s of src.logs) { -// let log = new Log(block.header, s.logIndex, s.transactionHash) - -// if (s.address != null) { -// log.address = s.address -// } - -// if (s.data != null) { -// log.data = s.data -// } - -// if (s.topics != null) { -// log.topics = s.topics -// } - -// block.logs.push(log) -// } -// } - -// setUpItems(block) -// return block -// } -// } diff --git a/tron/tron-stream/src/http/filter.ts b/tron/tron-stream/src/http/filter.ts new file mode 100644 index 000000000..fe75b0b95 --- /dev/null +++ b/tron/tron-stream/src/http/filter.ts @@ -0,0 +1,187 @@ +import {EntityFilter, FilterBuilder} from '@subsquid/util-internal-processor-tools' +import {assertNotNull, groupBy, weakMemo} from '@subsquid/util-internal' +import {getRequestAt, RangeRequest} from '@subsquid/util-internal-range' +import {Block, InternalTransaction, Log, Transaction} from '@subsquid/tron-normalization' +import {DataRequest, LogRequestRelations, TransactionRequestRelations, InternalTransactionRequestRelations} from '../data/data-request' + + +class IncludeSet { + public readonly logs = new Set() + public readonly transactions = new Set() + public readonly internalTransactions = new Set() + + addLog(log?: Log): void { + if (log) { + this.logs.add(log) + } + } + + addTransaction(tx?: Transaction): void { + if (tx) { + this.transactions.add(tx) + } + } + + addInternalTransaction(internalTx?: InternalTransaction): void { + if (internalTx) { + this.internalTransactions.add(internalTx) + } + } +} + + +function buildLogFilter(dataRequest: DataRequest): EntityFilter { + let logs = new EntityFilter() + + dataRequest.logs?.forEach(req => { + let where = req.where || {} + let filter = new FilterBuilder() + filter.propIn('address', where.address) + filter.getIn(log => assertNotNull(log.topics)[0], where.topic0) + filter.getIn(log => assertNotNull(log.topics)[1], where.topic1) + filter.getIn(log => assertNotNull(log.topics)[2], where.topic2) + filter.getIn(log => assertNotNull(log.topics)[3], where.topic3) + logs.add(filter, req.include ?? {}) + }) + + return logs +} + + +function buildTransactionFilter(dataRequest: DataRequest): EntityFilter { + let transactions = new EntityFilter() + + dataRequest.transactions?.forEach(req => { + let where = req.where || {} + let filter = new FilterBuilder() + filter.propIn('type', where.type) + transactions.add(filter, req.include ?? {}) + }) + + dataRequest.transferTransactions?.forEach(req => { + let where = req.where || {} + let filter = new FilterBuilder() + filter.propIn('type', ['TransferContract']) + filter.propIn('_transferContractTo', where.to) + filter.propIn('_transferContractOwner', where.owner) + transactions.add(filter, req.include ?? {}) + }) + + dataRequest.transferAssetTransactions?.forEach(req => { + let where = req.where || {} + let filter = new FilterBuilder() + filter.propIn('type', ['TransferAssetContract']) + filter.propIn('_transferAssetContractAsset', where.asset) + filter.propIn('_transferAssetContractOwner', where.owner) + filter.propIn('_transferAssetContractTo', where.to) + transactions.add(filter, req.include ?? {}) + }) + + dataRequest.triggerSmartContractTransactions?.forEach(req => { + let where = req.where || {} + let filter = new FilterBuilder() + filter.propIn('type', ['TriggerSmartContract']) + filter.propIn('_triggerSmartContractContract', where.contract) + filter.propIn('_triggerSmartContractOwner', where.owner) + filter.propIn('_triggerSmartContractSighash', where.sighash) + transactions.add(filter, req.include ?? {}) + }) + + return transactions +} + + +function buildInternalTransactionFilter(dataRequest: DataRequest): EntityFilter { + let inputs = new EntityFilter() + + dataRequest.internalTransactions?.forEach(req => { + let where = req.where || {} + let filter = new FilterBuilder() + filter.propIn('callerAddress', where.caller) + filter.propIn('transferToAddress', where.transferTo) + inputs.add(filter, req.include ?? {}) + }) + + return inputs +} + + +const getItemFilter = weakMemo((dataRequest: DataRequest) => { + return { + logs: buildLogFilter(dataRequest), + transactions: buildTransactionFilter(dataRequest), + internalTransaction: buildInternalTransactionFilter(dataRequest), + } +}) + + +export function filterBlock(block: Block, dataRequest: DataRequest): void { + let items = getItemFilter(dataRequest) + + let include = new IncludeSet() + + let transactions = new Map(block.transactions.map(tx => [tx.transactionIndex, tx])) + let internalTxByTx = groupBy(block.internalTransactions, input => input.transactionIndex) + let logsByTx = groupBy(block.logs, log => log.transactionIndex) + + if (items.logs.present()) { + for (let log of block.logs) { + let rel = items.logs.match(log) + if (rel == null) continue + include.addLog(log) + if (rel.transaction) { + let tx = assertNotNull(transactions.get(log.transactionIndex)) + include.addTransaction(tx) + } + } + } + + if (items.transactions.present()) { + for (let tx of block.transactions) { + let rel = items.transactions.match(tx) + if (rel == null) continue + include.addTransaction(tx) + if (rel.logs) { + let logs = assertNotNull(logsByTx.get(tx.transactionIndex)) + for (let log of logs) { + include.addLog(log) + } + } + if (rel.internalTransactions) { + let internalTxs = assertNotNull(internalTxByTx.get(tx.transactionIndex)) + for (let internalTx of internalTxs) { + include.addInternalTransaction(internalTx) + } + } + } + } + + if (items.internalTransaction.present()) { + for (let internalTx of block.internalTransactions) { + let rel = items.internalTransaction.match(internalTx) + if (rel == null) continue + include.addInternalTransaction(internalTx) + if (rel.transaction) { + let tx = assertNotNull(transactions.get(internalTx.transactionIndex)) + include.addTransaction(tx) + } + } + } + + block.logs = block.logs.filter(log => include.logs.has(log)) + block.transactions = block.transactions.filter(tx => include.transactions.has(tx)) + block.internalTransactions = block.internalTransactions.filter(internalTx => { + return include.internalTransactions.has(internalTx) + }) +} + + +export function filterBlockBatch(requests: RangeRequest[], blocks: Block[]): void { + for (let block of blocks) { + let dataRequest = getRequestAt(requests, block.header.height) || NO_DATA_REQUEST + filterBlock(block, dataRequest) + } +} + + +const NO_DATA_REQUEST: DataRequest = {} diff --git a/tron/tron-stream/src/http.ts b/tron/tron-stream/src/http/source.ts similarity index 55% rename from tron/tron-stream/src/http.ts rename to tron/tron-stream/src/http/source.ts index 2c7809253..3a8afd7f9 100644 --- a/tron/tron-stream/src/http.ts +++ b/tron/tron-stream/src/http/source.ts @@ -5,9 +5,9 @@ import { HttpDataSource as RawHttpDataSource } from '@subsquid/tron-data' import {mapBlock} from '@subsquid/tron-normalization' -import {DataRequest} from './data/data-request' -import {PartialBlock} from './data/data-partial' -// import {filterBlockBatch} from './filter' +import {DataRequest} from '../data/data-request' +import {PartialBlock} from '../data/data-partial' +import {filterBlockBatch} from './filter' export class HttpDataSource { @@ -35,7 +35,7 @@ export class HttpDataSource { stopOnHead )) { let blocks = batch.blocks.map(mapBlock) - // filterBlockBatch(requests, blocks) + filterBlockBatch(requests, blocks) yield blocks } } @@ -44,7 +44,17 @@ export class HttpDataSource { function toRawDataRequest(req: DataRequest): RawDataRequest { return { - transactions: false, - transactionsInfo: false + transactions: !!req.transactions?.length || + !!req.logs?.length || + !!req.internalTransactions?.length || + !!req.transferTransactions?.length || + !!req.transferAssetTransactions?.length || + !!req.triggerSmartContractTransactions?.length, + transactionsInfo: !!req.logs?.length || + !!req.internalTransactions?.length || + !!req.transactions?.some(req => req.include?.internalTransactions || req.include?.logs) || + !!req.transferTransactions?.some(req => req.include?.internalTransactions || req.include?.logs) || + !!req.transferAssetTransactions?.some(req => req.include?.internalTransactions || req.include?.logs) || + !!req.triggerSmartContractTransactions?.some(req => req.include?.internalTransactions || req.include?.logs) } } diff --git a/tron/tron-stream/src/index.ts b/tron/tron-stream/src/index.ts index 77fafebd2..5af423550 100644 --- a/tron/tron-stream/src/index.ts +++ b/tron/tron-stream/src/index.ts @@ -1,3 +1,2 @@ -export {assertNotNull} from '@subsquid/util-internal' +export * from './data/model' export * from './source' -// export * from './interfaces/data' diff --git a/tron/tron-stream/src/mapping.ts b/tron/tron-stream/src/mapping.ts deleted file mode 100644 index 7f67bdf8f..000000000 --- a/tron/tron-stream/src/mapping.ts +++ /dev/null @@ -1,224 +0,0 @@ -// import {TransactionResult} from '@subsquid/tron-data' -// import {HashAndHeight} from '@subsquid/util-internal-processor-tools' -// import {PartialBlockHeader} from './interfaces/data-partial' - - -// export class BlockHeader implements PartialBlockHeader { -// id: string -// height!: number -// hash!: string -// parentHash!: string -// txTrieRoot?: string -// version?: number -// timestamp?: number -// witnessAddress?: string -// witnessSignature?: string - -// constructor(src: PartialBlockHeader) { -// this.id = formatId(src) -// Object.assign(this, src) -// } -// } - - -// export class Transaction { -// hash: string -// ret?: TransactionResult[] -// signature?: string[] -// type?: string -// parameter?: any -// permissionId?: number -// refBlockBytes?: string -// refBlockHash?: string -// feeLimit?: number -// expiration?: number -// timestamp?: number -// rawDataHex?: string -// fee?: number -// contractResult?: string -// contractAddress?: string -// resMessage?: string -// withdrawAmount?: number -// unfreezeAmount?: number -// withdrawExpireAmount?: number -// cancelUnfreezeV2Amount: any -// result?: string -// energyFee?: number -// energyUsage?: number -// energyUsageTotal?: number -// netUsage?: number -// netFee?: number -// originEnergyUsage?: number -// energyPenaltyTotal?: number -// #block: BlockHeader -// #logs?: Log[] -// #internalTransactions?: InternalTransaction[] - -// constructor( -// block: BlockHeader, -// hash: string -// ) { -// this.hash = hash -// this.#block = block -// } - -// get block(): BlockHeader { -// return this.#block -// } - -// set block(value: BlockHeader) { -// this.#block = value -// } - -// get logs(): Log[] { -// return this.#logs || [] -// } - -// set logs(logs: Log[]) { -// this.#logs = logs -// } - -// get internalTransactions(): InternalTransaction[] { -// return this.#internalTransactions || [] -// } - -// set internalTransactions(internalTransactions: InternalTransaction[]) { -// this.#internalTransactions = internalTransactions -// } -// } - - -// export class InternalTransaction { -// transactionHash: string -// hash?: string -// callerAddress?: string -// transferToAddress?: string -// callValueInfo?: { -// callValue?: number -// tokenId?: string -// }[] -// note?: string -// rejected?: boolean -// extra?: string -// #block: BlockHeader -// #transaction?: Transaction - -// constructor( -// block: BlockHeader, -// transactionHash: string -// ) { -// this.transactionHash = transactionHash -// this.#block = block -// } - -// get block(): BlockHeader { -// return this.#block -// } - -// set block(value: BlockHeader) { -// this.#block = value -// } - -// get transaction(): Transaction | undefined { -// return this.#transaction -// } - -// set transaction(value: Transaction | undefined) { -// this.#transaction = value -// } - -// getTransaction(): Transaction { -// if (this.#transaction == null) { -// throw new Error(`Extrinsic is not set on internal transaction ${this.hash}`) -// } else { -// return this.#transaction -// } -// } -// } - - -// export class Log { -// id: string -// logIndex: number -// transactionHash: string -// address?: string -// data?: string -// topics?: string[] -// #block: BlockHeader -// #transaction?: Transaction - -// constructor( -// block: BlockHeader, -// logIndex: number, -// transactionHash: string -// ) { -// this.id = formatId(block, logIndex) -// this.logIndex = logIndex -// this.transactionHash = transactionHash -// this.#block = block -// } - -// get block(): BlockHeader { -// return this.#block -// } - -// set block(value: BlockHeader) { -// this.#block = value -// } - -// get transaction(): Transaction | undefined { -// return this.#transaction -// } - -// set transaction(value: Transaction | undefined) { -// this.#transaction = value -// } - -// getTransaction(): Transaction { -// if (this.#transaction == null) { -// throw new Error(`Transaction is not set on log ${this.id}`) -// } else { -// return this.#transaction -// } -// } -// } - - -// export class Block { -// constructor(public header: BlockHeader) {} -// transactions: Transaction[] = [] -// internalTransactions: InternalTransaction[] = [] -// logs: Log[] = [] -// } - - -// export function setUpItems(block: Block): void { -// let txByHash = new Map(block.transactions.map(tx => [tx.hash, tx])) - -// for (let i = 0; i < block.internalTransactions.length; i++) { -// let internalTx = block.internalTransactions[i] -// let tx = txByHash.get(internalTx.transactionHash) -// if (tx) { -// internalTx.transaction = tx -// } -// } - -// for (let log of block.logs) { -// let tx = txByHash.get(log.transactionHash) -// if (tx) { -// tx.logs.push(log) -// log.transaction = tx -// } -// } -// } - - -// function formatId(block: HashAndHeight, ...address: number[]): string { -// let no = block.height.toString().padStart(10, '0') -// let hash = block.hash.slice(16).slice(0, 5) -// let id = `${no}-${hash}` -// for (let index of address) { -// id += '-' + index.toString().padStart(6, '0') -// } -// return id -// } diff --git a/tron/tron-stream/src/source.ts b/tron/tron-stream/src/source.ts index 11cd6d11e..4d1a69a09 100644 --- a/tron/tron-stream/src/source.ts +++ b/tron/tron-stream/src/source.ts @@ -18,7 +18,7 @@ import { HttpDataSource as RawHttpDataSource } from '@subsquid/tron-data' import assert from 'assert' -import {HttpDataSource} from './http' +import {HttpDataSource} from './http/source' import {TronGateway} from './gateway/source' import {Block, FieldSelection} from './data/model' import {getFields} from './data/fields' @@ -390,7 +390,10 @@ class TronDataSource implements DataSource { } private createHttpDataSource(settings: HttpApiSettings): HttpDataSource { - let client = new HttpClient({baseUrl: settings.url}) + let client = new HttpClient({ + baseUrl: settings.url, + retryAttempts: Number.MAX_SAFE_INTEGER + }) let dataSource = new RawHttpDataSource({ httpApi: new HttpApi(client), strideConcurrency: settings.strideConcurrency,