From da65a08abe4a8b50995ac1804a71c601b89d3667 Mon Sep 17 00:00:00 2001 From: tmcgroul Date: Wed, 6 Mar 2024 20:01:39 +0700 Subject: [PATCH 1/5] implement evm-dump --- common/config/rush/pnpm-lock.yaml | 12 +++++ evm/evm-dump/bin/run.js | 3 ++ evm/evm-dump/package.json | 31 +++++++++++++ evm/evm-dump/src/dumper.ts | 74 +++++++++++++++++++++++++++++++ evm/evm-dump/src/main.ts | 3 ++ evm/evm-dump/tsconfig.json | 21 +++++++++ rush.json | 6 +++ 7 files changed, 150 insertions(+) create mode 100644 evm/evm-dump/bin/run.js create mode 100644 evm/evm-dump/package.json create mode 100644 evm/evm-dump/src/dumper.ts create mode 100644 evm/evm-dump/src/main.ts create mode 100644 evm/evm-dump/tsconfig.json diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 78811b9c0..a30fbb12f 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -47,6 +47,9 @@ dependencies: '@rush-temp/erc20-transfers': specifier: file:./projects/erc20-transfers.tgz version: file:projects/erc20-transfers.tgz(supports-color@8.1.1)(ts-node@10.9.2) + '@rush-temp/evm-dump': + specifier: file:./projects/evm-dump.tgz + version: file:projects/evm-dump.tgz '@rush-temp/evm-processor': specifier: file:./projects/evm-processor.tgz version: file:projects/evm-processor.tgz @@ -5268,6 +5271,15 @@ packages: - utf-8-validate dev: false + file:projects/evm-dump.tgz: + resolution: {integrity: sha512-DKBph5BNVGkG2JASJLUQXh4WBWlzQINGgYR91CwAJQREV3B99eDUY+ywr0O5FVJxxWCOkvWj41hExKqLQWtmSQ==, tarball: file:projects/evm-dump.tgz} + name: '@rush-temp/evm-dump' + version: 0.0.0 + dependencies: + '@types/node': 18.19.4 + typescript: 5.3.3 + dev: false + file:projects/evm-processor.tgz: resolution: {integrity: sha512-ZeY6rzzfGzR19tgMLcmmUZ+P3FMJ09o8V3hn/p2ESBJJef3LKfXzO3weomLwG5VE6IHeRUxh2VBqIe8sShLi0g==, tarball: file:projects/evm-processor.tgz} name: '@rush-temp/evm-processor' diff --git a/evm/evm-dump/bin/run.js b/evm/evm-dump/bin/run.js new file mode 100644 index 000000000..345e3cf6f --- /dev/null +++ b/evm/evm-dump/bin/run.js @@ -0,0 +1,3 @@ +#!/usr/bin/env node + +require('../lib/main') diff --git a/evm/evm-dump/package.json b/evm/evm-dump/package.json new file mode 100644 index 000000000..a8c4fed55 --- /dev/null +++ b/evm/evm-dump/package.json @@ -0,0 +1,31 @@ +{ + "name": "@subsquid/evm-dump", + "version": "0.0.0", + "description": "Data archiving tool for EVM", + "license": "GPL-3.0-or-later", + "repository": "git@github.com:subsquid/squid.git", + "publishConfig": { + "access": "public" + }, + "bin": { + "evm-dump": "./bin/run.js" + }, + "files": [ + "bin", + "lib", + "src" + ], + "scripts": { + "build": "rm -rf lib && tsc" + }, + "dependencies": { + "@subsquid/evm-processor": "^1.17.0", + "@subsquid/util-internal": "^3.0.0", + "@subsquid/util-internal-ingest-tools": "^1.1.1", + "@subsquid/util-internal-dump-cli": "^0.0.0" + }, + "devDependencies": { + "@types/node": "^18.18.14", + "typescript": "~5.3.2" + } +} diff --git a/evm/evm-dump/src/dumper.ts b/evm/evm-dump/src/dumper.ts new file mode 100644 index 000000000..a86f534df --- /dev/null +++ b/evm/evm-dump/src/dumper.ts @@ -0,0 +1,74 @@ +import {Block, DataRequest} from '@subsquid/evm-processor/lib/ds-rpc/rpc-data' +import {Rpc} from '@subsquid/evm-processor/lib/ds-rpc/rpc' +import {def} from '@subsquid/util-internal' +import {coldIngest} from '@subsquid/util-internal-ingest-tools' +import {Command, Dumper, DumperOptions, positiveInt, Range} from '@subsquid/util-internal-dump-cli' + + +interface Options extends DumperOptions { + bestBlockOffset: number + withReceipts: boolean + withTraces: boolean + withStatediffs: boolean + useTraceApi: boolean + useDebugApiForStatediffs: boolean +} + + +export class EvmDumper extends Dumper { + protected setUpProgram(program: Command): void { + program.description('Data archiving tool for EVM') + program.option('--best-block-offset ', 'Finality offset from the head of chain', positiveInt, 30) + program.option('--with-receipts', 'Fetch transaction receipt data') + program.option('--with-traces', 'Fetch EVM call traces') + program.option('--with-statediffs', 'Fetch EVM state updates') + program.option('--use-trace-api', 'Use trace_* API for statediffs and call traces') + program.option('--use-debug-api-for-statediffs', 'Use debug prestateTracer to fetch statediffs (by default will use trace_* api)') + } + + protected getLoggingNamespace(): string { + return 'sqd:evm-dump' + } + + protected getPrevBlockHash(block: Block): string { + return block.block.parentHash + } + + protected getDefaultTopDirSize(): number { + return 500 + } + + @def + private getDataSource(): Rpc { + return new Rpc(this.rpc()) + } + + protected async* getBlocks(range: Range): AsyncIterable { + let request: DataRequest = { + logs: true, + transactions: true, + receipts: this.options().withReceipts, + traces: this.options().withTraces, + stateDiffs: this.options().withStatediffs, + preferTraceApi: this.options().useTraceApi, + useDebugApiForStateDiffs: this.options().useDebugApiForStatediffs + } + let batches = coldIngest({ + getFinalizedHeight: () => this.getFinalizedHeight(), + getSplit: req => this.getDataSource().getColdSplit(req), + requests: [{range, request}], + concurrency: Math.min(5, this.getDataSource().client.getConcurrency()), + splitSize: 10, + stopOnHead: false, + headPollInterval: 10_000 + }) + for await (let batch of batches) { + yield batch.blocks + } + } + + protected async getFinalizedHeight(): Promise { + let height = await this.getDataSource().getHeight() + return Math.max(0, height - this.options().bestBlockOffset) + } +} diff --git a/evm/evm-dump/src/main.ts b/evm/evm-dump/src/main.ts new file mode 100644 index 000000000..48d1c6048 --- /dev/null +++ b/evm/evm-dump/src/main.ts @@ -0,0 +1,3 @@ +import {EvmDumper} from './dumper' + +new EvmDumper().run() diff --git a/evm/evm-dump/tsconfig.json b/evm/evm-dump/tsconfig.json new file mode 100644 index 000000000..deee9f66b --- /dev/null +++ b/evm/evm-dump/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "module": "commonjs", + "target": "es2020", + "outDir": "lib", + "rootDir": "src", + "allowJs": true, + "strict": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "esModuleInterop": true, + "experimentalDecorators": true, + "emitDecoratorMetadata": true, + "skipLibCheck": true + }, + "include": ["src"], + "exclude": [ + "node_modules" + ] +} diff --git a/rush.json b/rush.json index 2ba6a6694..dfacf1dfb 100644 --- a/rush.json +++ b/rush.json @@ -450,6 +450,12 @@ // */ // // "versionPolicyName": "" // }, + { + "packageName": "@subsquid/evm-dump", + "projectFolder": "evm/evm-dump", + "shouldPublish": true, + "versionPolicyName": "npm" + }, { "packageName": "@subsquid/evm-processor", "projectFolder": "evm/evm-processor", From 7551cd489dd7c7692d7bbbc0c4015a2b87826818 Mon Sep 17 00:00:00 2001 From: tmcgroul Date: Sat, 9 Mar 2024 14:25:17 +0700 Subject: [PATCH 2/5] refactor evm-processor --- common/config/rush/pnpm-lock.yaml | 28 +- evm/evm-data/package.json | 34 ++ .../src/interfaces => evm-data/src}/base.ts | 0 evm/evm-data/src/index.ts | 1 + evm/evm-data/src/normalization/data.ts | 205 +++++++++++ evm/evm-data/src/normalization/index.ts | 2 + evm/evm-data/src/normalization/mapping.ts | 110 ++++++ evm/evm-data/src/rpc/index.ts | 2 + .../ds-rpc => evm-data/src/rpc}/rpc-data.ts | 79 +++- .../src/ds-rpc => evm-data/src/rpc}/rpc.ts | 11 +- evm/evm-data/src/rpc/util.ts | 23 ++ evm/evm-data/tsconfig.json | 21 ++ evm/evm-dump/package.json | 2 +- evm/evm-dump/src/dumper.ts | 4 +- evm/evm-ingest/bin/run.js | 3 + evm/evm-ingest/package.json | 31 ++ evm/evm-ingest/src/ingest.ts | 37 ++ evm/evm-ingest/src/main.ts | 3 + evm/evm-ingest/tsconfig.json | 21 ++ evm/evm-processor/package.json | 1 + evm/evm-processor/src/ds-archive/client.ts | 2 +- evm/evm-processor/src/ds-rpc/client.ts | 4 +- evm/evm-processor/src/ds-rpc/mapping.ts | 10 +- evm/evm-processor/src/ds-rpc/request.ts | 4 +- evm/evm-processor/src/ds-rpc/schema.ts | 4 +- evm/evm-processor/src/ds-rpc/util.ts | 15 +- .../src/interfaces/data-request.ts | 4 +- evm/evm-processor/src/interfaces/data.ts | 2 +- evm/evm-processor/src/interfaces/evm.ts | 344 +++++++++--------- evm/evm-processor/src/mapping/entities.ts | 6 +- evm/evm-processor/src/mapping/schema.ts | 12 +- evm/evm-processor/src/mapping/selection.ts | 12 +- rush.json | 12 + 33 files changed, 818 insertions(+), 231 deletions(-) create mode 100644 evm/evm-data/package.json rename evm/{evm-processor/src/interfaces => evm-data/src}/base.ts (100%) create mode 100644 evm/evm-data/src/index.ts create mode 100644 evm/evm-data/src/normalization/data.ts create mode 100644 evm/evm-data/src/normalization/index.ts create mode 100644 evm/evm-data/src/normalization/mapping.ts create mode 100644 evm/evm-data/src/rpc/index.ts rename evm/{evm-processor/src/ds-rpc => evm-data/src/rpc}/rpc-data.ts (71%) rename evm/{evm-processor/src/ds-rpc => evm-data/src/rpc}/rpc.ts (99%) create mode 100644 evm/evm-data/src/rpc/util.ts create mode 100644 evm/evm-data/tsconfig.json create mode 100644 evm/evm-ingest/bin/run.js create mode 100644 evm/evm-ingest/package.json create mode 100644 evm/evm-ingest/src/ingest.ts create mode 100644 evm/evm-ingest/src/main.ts create mode 100644 evm/evm-ingest/tsconfig.json diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index a30fbb12f..4fb8e1c4a 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -47,9 +47,15 @@ dependencies: '@rush-temp/erc20-transfers': specifier: file:./projects/erc20-transfers.tgz version: file:projects/erc20-transfers.tgz(supports-color@8.1.1)(ts-node@10.9.2) + '@rush-temp/evm-data': + specifier: file:./projects/evm-data.tgz + version: file:projects/evm-data.tgz '@rush-temp/evm-dump': specifier: file:./projects/evm-dump.tgz version: file:projects/evm-dump.tgz + '@rush-temp/evm-ingest': + specifier: file:./projects/evm-ingest.tgz + version: file:projects/evm-ingest.tgz '@rush-temp/evm-processor': specifier: file:./projects/evm-processor.tgz version: file:projects/evm-processor.tgz @@ -5271,8 +5277,17 @@ packages: - utf-8-validate dev: false + file:projects/evm-data.tgz: + resolution: {integrity: sha512-TIl6xxW97ji3e9TtkSxuwkMLeJmaTJMyI42oEcdEkqb45yfAY8MH4HdmOvo19xGk2tL7yFXFKTwV+n9+DniTWw==, tarball: file:projects/evm-data.tgz} + name: '@rush-temp/evm-data' + version: 0.0.0 + dependencies: + '@types/node': 18.19.4 + typescript: 5.3.3 + dev: false + file:projects/evm-dump.tgz: - resolution: {integrity: sha512-DKBph5BNVGkG2JASJLUQXh4WBWlzQINGgYR91CwAJQREV3B99eDUY+ywr0O5FVJxxWCOkvWj41hExKqLQWtmSQ==, tarball: file:projects/evm-dump.tgz} + resolution: {integrity: sha512-+Qck/LHUTEWDScuoLgYXWD55XINSFXqonGKhsNprG0ASZu6r1H3qJbHWa0uLlaxLErKNXCLKqK+gCmEqU9htQg==, tarball: file:projects/evm-dump.tgz} name: '@rush-temp/evm-dump' version: 0.0.0 dependencies: @@ -5280,8 +5295,17 @@ packages: typescript: 5.3.3 dev: false + file:projects/evm-ingest.tgz: + resolution: {integrity: sha512-/YUduCwLPjD4XmVuoaLTv0gkhKvyz1h2Oqen0m/S8GrtMdxiJE5KPUHTDe/D6F2RBPn34c743rM7QTh9GU9fdg==, tarball: file:projects/evm-ingest.tgz} + name: '@rush-temp/evm-ingest' + version: 0.0.0 + dependencies: + '@types/node': 18.19.4 + typescript: 5.3.3 + dev: false + file:projects/evm-processor.tgz: - resolution: {integrity: sha512-ZeY6rzzfGzR19tgMLcmmUZ+P3FMJ09o8V3hn/p2ESBJJef3LKfXzO3weomLwG5VE6IHeRUxh2VBqIe8sShLi0g==, tarball: file:projects/evm-processor.tgz} + resolution: {integrity: sha512-PiWYYUNdXL4iOe+/JQPa88LkRD5ebn8aq07ZHLdt9QOF8QKvOnkMNzIIBPCIiHoVKKUtJ5WWmgAZnjX4WxhmMA==, tarball: file:projects/evm-processor.tgz} name: '@rush-temp/evm-processor' version: 0.0.0 dependencies: diff --git a/evm/evm-data/package.json b/evm/evm-data/package.json new file mode 100644 index 000000000..d75d1e02e --- /dev/null +++ b/evm/evm-data/package.json @@ -0,0 +1,34 @@ +{ + "name": "@subsquid/evm-data", + "version": "0.0.0", + "description": "EVM data definition and fetching", + "license": "GPL-3.0-or-later", + "repository": "git@github.com:subsquid/squid.git", + "publishConfig": { + "access": "public" + }, + "files": [ + "lib", + "src" + ], + "main": "lib/index.js", + "scripts": { + "build": "rm -rf lib && tsc", + "test": "node lib/test.js" + }, + "dependencies": { + "@subsquid/logger": "^1.3.3", + "@subsquid/util-internal": "^3.0.0", + "@subsquid/util-internal-ingest-tools": "^1.1.1", + "@subsquid/util-internal-range": "^0.2.0", + "@subsquid/util-internal-validation": "^0.3.0" + }, + "peerDependencies": { + "@subsquid/rpc-client": "^4.6.0" + }, + "devDependencies": { + "@subsquid/rpc-client": "^4.6.0", + "@types/node": "^18.18.14", + "typescript": "~5.3.2" + } +} \ No newline at end of file diff --git a/evm/evm-processor/src/interfaces/base.ts b/evm/evm-data/src/base.ts similarity index 100% rename from evm/evm-processor/src/interfaces/base.ts rename to evm/evm-data/src/base.ts diff --git a/evm/evm-data/src/index.ts b/evm/evm-data/src/index.ts new file mode 100644 index 000000000..85e56520d --- /dev/null +++ b/evm/evm-data/src/index.ts @@ -0,0 +1 @@ +export * from './base' diff --git a/evm/evm-data/src/normalization/data.ts b/evm/evm-data/src/normalization/data.ts new file mode 100644 index 000000000..fd86d4748 --- /dev/null +++ b/evm/evm-data/src/normalization/data.ts @@ -0,0 +1,205 @@ +import {Bytes, Bytes20, Bytes32, Bytes8} from '../base' + + +export interface EvmBlockHeader { + height: number + hash: Bytes32 + parentHash: Bytes32 + nonce?: Bytes8 + sha3Uncles: Bytes32 + logsBloom: Bytes + transactionsRoot: Bytes32 + stateRoot: Bytes32 + receiptsRoot: Bytes32 + mixHash?: Bytes + miner: Bytes20 + difficulty?: bigint + totalDifficulty?: bigint + extraData: Bytes + size: bigint + gasLimit: bigint + gasUsed: bigint + timestamp: number + baseFeePerGas?: bigint + /** + * This field is not supported by all currently deployed archives. + * Requesting it may cause internal error. + */ + l1BlockNumber?: number +} + + +export interface EvmTransaction extends _EvmTx, _EvmTxReceipt { + transactionIndex: number + sighash: Bytes +} + + +export interface _EvmTx { + hash: Bytes32 + from: Bytes20 + to?: Bytes20 + gas: bigint + gasPrice: bigint + maxFeePerGas?: bigint + maxPriorityFeePerGas?: bigint + input: Bytes + nonce: number + value: bigint + v?: bigint + r?: Bytes32 + s?: Bytes32 + yParity?: number + chainId?: number +} + + +export interface _EvmTxReceipt { + // gasUsed: bigint + // cumulativeGasUsed: bigint + // effectiveGasPrice: bigint + // contractAddress?: Bytes32 + // type: number + // status: number +} + + +export interface EvmLog { + logIndex: number + transactionIndex: number + transactionHash: Bytes32 + address: Bytes20 + data: Bytes + topics: Bytes32[] +} + + +export interface EvmTraceBase { + transactionIndex: number + traceAddress: number[] + subtraces: number + error: string | null + revertReason?: string +} + + +export interface EvmTraceCreate extends EvmTraceBase { + type: 'create' + action: EvmTraceCreateAction + result?: EvmTraceCreateResult +} + + +export interface EvmTraceCreateAction { + from: Bytes20 + value: bigint + gas: bigint + init: Bytes +} + + +export interface EvmTraceCreateResult { + gasUsed: bigint + code: Bytes + address: Bytes20 +} + + +export interface EvmTraceCall extends EvmTraceBase { + type: 'call' + action: EvmTraceCallAction + result?: EvmTraceCallResult +} + + +export interface EvmTraceCallAction { + callType: string + from: Bytes20 + to: Bytes20 + value?: bigint + gas: bigint + input: Bytes + sighash: Bytes +} + + +export interface EvmTraceCallResult { + gasUsed: bigint + output: Bytes +} + + +export interface EvmTraceSuicide extends EvmTraceBase { + type: 'suicide' + action: EvmTraceSuicideAction +} + + +export interface EvmTraceSuicideAction { + address: Bytes20 + refundAddress: Bytes20 + balance: bigint +} + + +export interface EvmTraceReward extends EvmTraceBase { + type: 'reward' + action: EvmTraceRewardAction +} + + +export interface EvmTraceRewardAction { + author: Bytes20 + value: bigint + type: string +} + + +export type EvmTrace = EvmTraceCreate | EvmTraceCall | EvmTraceSuicide | EvmTraceReward + + +export interface EvmStateDiffBase { + transactionIndex: number + address: Bytes20 + key: 'balance' | 'code' | 'nonce' | Bytes32 +} + + +export interface EvmStateDiffNoChange extends EvmStateDiffBase { + kind: '=' + prev?: null + next?: null +} + + +export interface EvmStateDiffAdd extends EvmStateDiffBase { + kind: '+' + prev?: null + next: Bytes +} + + +export interface EvmStateDiffChange extends EvmStateDiffBase { + kind: '*' + prev: Bytes + next: Bytes +} + + +export interface EvmStateDiffDelete extends EvmStateDiffBase { + kind: '-' + prev: Bytes + next?: null +} + + +export type EvmStateDiff = EvmStateDiffNoChange | EvmStateDiffAdd | EvmStateDiffChange | EvmStateDiffDelete + + +export interface Block { + header: EvmBlockHeader + transactions: EvmTransaction[] + logs: EvmLog[] + traces: EvmTrace[] + stateDiffs: EvmStateDiff[] +} diff --git a/evm/evm-data/src/normalization/index.ts b/evm/evm-data/src/normalization/index.ts new file mode 100644 index 000000000..808f72cb7 --- /dev/null +++ b/evm/evm-data/src/normalization/index.ts @@ -0,0 +1,2 @@ +export * from './data' +export * from './mapping' diff --git a/evm/evm-data/src/normalization/mapping.ts b/evm/evm-data/src/normalization/mapping.ts new file mode 100644 index 000000000..c62739b9b --- /dev/null +++ b/evm/evm-data/src/normalization/mapping.ts @@ -0,0 +1,110 @@ +import {addErrorContext, groupBy, unexpectedCase} from '@subsquid/util-internal' +import assert from 'assert' +import * as rpc from '../rpc' +import {Block, EvmBlockHeader, EvmTransaction, EvmLog, EvmTrace, EvmStateDiff} from './data' + + +export function mapRpcBlock(src: rpc.Block): Block { + let header: EvmBlockHeader = { + hash: src.hash, + height: src.height, + parentHash: src.block.parentHash, + logsBloom: src.block.logsBloom, + extraData: src.block.extraData, + gasLimit: BigInt(src.block.gasLimit), + gasUsed: BigInt(src.block.gasUsed), + miner: src.block.miner, + receiptsRoot: src.block.receiptsRoot, + sha3Uncles: src.block.sha3Uncles, + size: BigInt(src.block.size), + stateRoot: src.block.stateRoot, + timestamp: parseInt(src.block.timestamp), + transactionsRoot: src.block.transactionsRoot, + } + + if (src.block.baseFeePerGas) { + header.baseFeePerGas = BigInt(src.block.baseFeePerGas) + } + if (src.block.difficulty) { + header.difficulty = BigInt(src.block.difficulty) + } + if (src.block.totalDifficulty) { + header.totalDifficulty = BigInt(src.block.totalDifficulty) + } + if (src.block.l1BlockNumber) { + header.l1BlockNumber = parseInt(src.block.l1BlockNumber) + } + if (src.block.mixHash) { + header.mixHash = src.block.mixHash + } + if (src.block.nonce) { + header.nonce = src.block.nonce + } + + let transactions: EvmTransaction[] = [] + let logs: EvmLog[] = [] + let traces: EvmTrace[] = [] + let stateDiffs: EvmStateDiff[] = [] + + for (let tx of src.block.transactions) { + if (typeof tx == 'string') continue + let transaction = mapRpcTransaction(tx) + transactions.push(transaction) + } + + return { + header, + transactions, + logs, + traces, + stateDiffs + } +} + + +function mapRpcTransaction(src: rpc.Transaction): EvmTransaction { + let tx: EvmTransaction = { + transactionIndex: parseInt(src.transactionIndex), + sighash: src.input.slice(0, 10), + hash: src.hash, + from: src.from, + gas: BigInt(src.gas), + gasPrice: BigInt(src.gasPrice), + input: src.input, + nonce: parseInt(src.nonce), + value: BigInt(src.value), + // gasUsed: bigint + // cumulativeGasUsed: bigint + // effectiveGasPrice: bigint + // contractAddress?: Bytes32 + // type: number + // status: number + } + + if (src.to) { + tx.to = src.to + } + if (src.maxFeePerGas) { + tx.maxFeePerGas = BigInt(src.maxFeePerGas) + } + if (src.maxPriorityFeePerGas) { + tx.maxPriorityFeePerGas = BigInt(src.maxPriorityFeePerGas) + } + if (src.v) { + tx.v = BigInt(src.v) + } + if (src.r) { + tx.r = src.r + } + if (src.s) { + tx.s = src.s + } + if (src.yParity) { + tx.yParity = parseInt(src.yParity) + } + if (src.chainId) { + tx.chainId = parseInt(src.chainId) + } + + return tx +} diff --git a/evm/evm-data/src/rpc/index.ts b/evm/evm-data/src/rpc/index.ts new file mode 100644 index 000000000..460c6f4e1 --- /dev/null +++ b/evm/evm-data/src/rpc/index.ts @@ -0,0 +1,2 @@ +export * from './rpc-data' +export * from './rpc' diff --git a/evm/evm-processor/src/ds-rpc/rpc-data.ts b/evm/evm-data/src/rpc/rpc-data.ts similarity index 71% rename from evm/evm-processor/src/ds-rpc/rpc-data.ts rename to evm/evm-data/src/rpc/rpc-data.ts index 368ff069a..8641f8618 100644 --- a/evm/evm-processor/src/ds-rpc/rpc-data.ts +++ b/evm/evm-data/src/rpc/rpc-data.ts @@ -15,8 +15,23 @@ import { STRING, Validator } from '@subsquid/util-internal-validation' -import {Bytes, Bytes20, Bytes32, Qty} from '../interfaces/base' -import {project} from '../mapping/schema' +import {Bytes, Bytes8, Bytes20, Bytes32, Qty} from '../base' + + +export function project( + fields: F | undefined, + obj: T +): Partial { + if (fields == null) return {} + let result: Partial = {} + let key: keyof T + for (key in obj) { + if (fields[key]) { + result[key] = obj[key] + } + } + return result +} export interface DataRequest { @@ -50,27 +65,59 @@ const Transaction = object({ blockHash: BYTES, transactionIndex: SMALL_QTY, hash: BYTES, - input: BYTES + input: BYTES, + nonce: SMALL_QTY, + from: BYTES, + to: option(BYTES), + value: QTY, + type: option(SMALL_QTY), + gas: QTY, + gasPrice: QTY, + maxFeePerGas: option(QTY), + maxPriorityFeePerGas: option(QTY), + v: option(QTY), + r: option(BYTES), + s: option(BYTES), + yParity: option(SMALL_QTY), + chainId: option(SMALL_QTY) }) export type Transaction = GetSrcType -export const GetBlockWithTransactions = object({ +const GetBlockBase = { number: SMALL_QTY, hash: BYTES, parentHash: BYTES, logsBloom: BYTES, + timestamp: SMALL_QTY, + transactionsRoot: BYTES, + receiptsRoot: BYTES, + stateRoot: BYTES, + sha3Uncles: BYTES, + extraData: BYTES, + miner: BYTES, + nonce: option(BYTES), + mixHash: option(BYTES), + size: SMALL_QTY, + gasLimit: QTY, + gasUsed: QTY, + difficulty: option(QTY), + totalDifficulty: option(QTY), + baseFeePerGas: option(QTY), + l1BlockNumber: option(SMALL_QTY) +} + + +export const GetBlockWithTransactions = object({ + ...GetBlockBase, transactions: array(Transaction) }) export const GetBlockNoTransactions = object({ - number: SMALL_QTY, - hash: BYTES, - parentHash: BYTES, - logsBloom: BYTES, + ...GetBlockBase, transactions: array(BYTES) }) @@ -80,6 +127,22 @@ export interface GetBlock { hash: Bytes32 parentHash: Bytes32 logsBloom: Bytes + timestamp: Qty + transactionsRoot: Bytes32 + receiptsRoot: Bytes32 + stateRoot: Bytes32 + sha3Uncles: Bytes32 + extraData: Bytes + miner: Bytes20 + nonce?: Bytes8 | null + mixHash?: Bytes | null + size: Qty + gasLimit: Qty + gasUsed: Qty + difficulty?: Qty | null + totalDifficulty?: Qty | null + baseFeePerGas?: Qty | null + l1BlockNumber?: Qty | null transactions: Bytes32[] | Transaction[] } diff --git a/evm/evm-processor/src/ds-rpc/rpc.ts b/evm/evm-data/src/rpc/rpc.ts similarity index 99% rename from evm/evm-processor/src/ds-rpc/rpc.ts rename to evm/evm-data/src/rpc/rpc.ts index eff5fb307..1921b5d6c 100644 --- a/evm/evm-processor/src/ds-rpc/rpc.ts +++ b/evm/evm-data/src/rpc/rpc.ts @@ -6,8 +6,7 @@ import {assertIsValid, BlockConsistencyError, trimInvalid} from '@subsquid/util- import {FiniteRange, rangeToArray, SplitRequest} from '@subsquid/util-internal-range' import {array, DataValidationError, GetSrcType, nullable, Validator} from '@subsquid/util-internal-validation' import assert from 'assert' -import {Bytes, Bytes32, Qty} from '../interfaces/base' -import {isEmpty} from '../mapping/schema' +import {Bytes, Bytes32, Qty} from '../base' import { Block, DataRequest, @@ -28,6 +27,14 @@ import {getTxHash, qty2Int, toQty} from './util' const NO_LOGS_BLOOM = '0x'+Buffer.alloc(256).toString('hex') +export function isEmpty(obj: object): boolean { + for (let _ in obj) { + return false + } + return true +} + + function getResultValidator(validator: V): (result: unknown) => GetSrcType { return function(result: unknown) { let err = validator.validate(result) diff --git a/evm/evm-data/src/rpc/util.ts b/evm/evm-data/src/rpc/util.ts new file mode 100644 index 000000000..8f0389f25 --- /dev/null +++ b/evm/evm-data/src/rpc/util.ts @@ -0,0 +1,23 @@ +import assert from 'assert' +import {Bytes32, Qty} from '../base' + + +export function qty2Int(qty: Qty): number { + let i = parseInt(qty, 16) + assert(Number.isSafeInteger(i)) + return i +} + + +export function toQty(i: number): Qty { + return '0x' + i.toString(16) +} + + +export function getTxHash(tx: Bytes32 | {hash: Bytes32}): Bytes32 { + if (typeof tx == 'string') { + return tx + } else { + return tx.hash + } +} diff --git a/evm/evm-data/tsconfig.json b/evm/evm-data/tsconfig.json new file mode 100644 index 000000000..deee9f66b --- /dev/null +++ b/evm/evm-data/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "module": "commonjs", + "target": "es2020", + "outDir": "lib", + "rootDir": "src", + "allowJs": true, + "strict": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "esModuleInterop": true, + "experimentalDecorators": true, + "emitDecoratorMetadata": true, + "skipLibCheck": true + }, + "include": ["src"], + "exclude": [ + "node_modules" + ] +} diff --git a/evm/evm-dump/package.json b/evm/evm-dump/package.json index a8c4fed55..44815e58e 100644 --- a/evm/evm-dump/package.json +++ b/evm/evm-dump/package.json @@ -19,7 +19,7 @@ "build": "rm -rf lib && tsc" }, "dependencies": { - "@subsquid/evm-processor": "^1.17.0", + "@subsquid/evm-data": "^0.0.0", "@subsquid/util-internal": "^3.0.0", "@subsquid/util-internal-ingest-tools": "^1.1.1", "@subsquid/util-internal-dump-cli": "^0.0.0" diff --git a/evm/evm-dump/src/dumper.ts b/evm/evm-dump/src/dumper.ts index a86f534df..025e6252c 100644 --- a/evm/evm-dump/src/dumper.ts +++ b/evm/evm-dump/src/dumper.ts @@ -1,5 +1,5 @@ -import {Block, DataRequest} from '@subsquid/evm-processor/lib/ds-rpc/rpc-data' -import {Rpc} from '@subsquid/evm-processor/lib/ds-rpc/rpc' +import {Block, DataRequest} from '@subsquid/evm-data/lib/rpc/rpc-data' +import {Rpc} from '@subsquid/evm-data/lib/rpc/rpc' import {def} from '@subsquid/util-internal' import {coldIngest} from '@subsquid/util-internal-ingest-tools' import {Command, Dumper, DumperOptions, positiveInt, Range} from '@subsquid/util-internal-dump-cli' diff --git a/evm/evm-ingest/bin/run.js b/evm/evm-ingest/bin/run.js new file mode 100644 index 000000000..345e3cf6f --- /dev/null +++ b/evm/evm-ingest/bin/run.js @@ -0,0 +1,3 @@ +#!/usr/bin/env node + +require('../lib/main') diff --git a/evm/evm-ingest/package.json b/evm/evm-ingest/package.json new file mode 100644 index 000000000..99fabcacc --- /dev/null +++ b/evm/evm-ingest/package.json @@ -0,0 +1,31 @@ +{ + "name": "@subsquid/evm-ingest", + "version": "0.0.0", + "description": "Data fetcher and normalizer for EVM", + "license": "GPL-3.0-or-later", + "repository": "git@github.com:subsquid/squid.git", + "publishConfig": { + "access": "public" + }, + "bin": { + "evm-ingest": "./bin/run.js" + }, + "files": [ + "bin", + "lib", + "src" + ], + "scripts": { + "build": "rm -rf lib && tsc" + }, + "dependencies": { + "@subsquid/evm-data": "^0.0.0", + "@subsquid/util-internal": "^3.0.0", + "@subsquid/util-internal-ingest-cli": "^0.0.0", + "@subsquid/util-internal-json": "^1.2.2" + }, + "devDependencies": { + "@types/node": "^18.18.14", + "typescript": "~5.3.2" + } +} diff --git a/evm/evm-ingest/src/ingest.ts b/evm/evm-ingest/src/ingest.ts new file mode 100644 index 000000000..eccb643a9 --- /dev/null +++ b/evm/evm-ingest/src/ingest.ts @@ -0,0 +1,37 @@ +import {mapRpcBlock} from '@subsquid/evm-data/lib/normalization' +import {Block as RawBlock} from '@subsquid/evm-data/lib/rpc' +import {addErrorContext} from '@subsquid/util-internal' +import {Command, Ingest, Range} from '@subsquid/util-internal-ingest-cli' +import {toJSON} from '@subsquid/util-internal-json' + + +export class EvmIngest extends Ingest { + protected getLoggingNamespace(): string { + return 'sqd:evm-ingest' + } + + protected setUpProgram(program: Command) { + program.description('Data ingestion tool for EVM') + program.options.forEach(option => { + if (option.attributeName() == 'rawArchive') { + option.required = true + } + }) + } + + protected async *getBlocks(range: Range): AsyncIterable { + for await (let blocks of this.archive().getRawBlocks(range)) { + yield blocks.map(raw => { + try { + let block = mapRpcBlock(raw) + return toJSON(block) + } catch(err: any) { + throw addErrorContext(err, { + blockHeight: raw.height, + blockHash: raw.hash + }) + } + }) + } + } +} diff --git a/evm/evm-ingest/src/main.ts b/evm/evm-ingest/src/main.ts new file mode 100644 index 000000000..f681fe68e --- /dev/null +++ b/evm/evm-ingest/src/main.ts @@ -0,0 +1,3 @@ +import {EvmIngest} from './ingest' + +new EvmIngest().run() diff --git a/evm/evm-ingest/tsconfig.json b/evm/evm-ingest/tsconfig.json new file mode 100644 index 000000000..deee9f66b --- /dev/null +++ b/evm/evm-ingest/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "module": "commonjs", + "target": "es2020", + "outDir": "lib", + "rootDir": "src", + "allowJs": true, + "strict": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "esModuleInterop": true, + "experimentalDecorators": true, + "emitDecoratorMetadata": true, + "skipLibCheck": true + }, + "include": ["src"], + "exclude": [ + "node_modules" + ] +} diff --git a/evm/evm-processor/package.json b/evm/evm-processor/package.json index 78648e82a..9e59c2cbe 100644 --- a/evm/evm-processor/package.json +++ b/evm/evm-processor/package.json @@ -20,6 +20,7 @@ "@subsquid/http-client": "^1.3.2", "@subsquid/logger": "^1.3.3", "@subsquid/rpc-client": "^4.6.0", + "@subsquid/evm-data": "^0.0.0", "@subsquid/util-internal": "^3.0.0", "@subsquid/util-internal-archive-client": "^0.1.1", "@subsquid/util-internal-hex": "^1.2.2", diff --git a/evm/evm-processor/src/ds-archive/client.ts b/evm/evm-processor/src/ds-archive/client.ts index 2dc033d67..85ca11f41 100644 --- a/evm/evm-processor/src/ds-archive/client.ts +++ b/evm/evm-processor/src/ds-archive/client.ts @@ -3,9 +3,9 @@ import {ArchiveClient} from '@subsquid/util-internal-archive-client' import {archiveIngest} from '@subsquid/util-internal-ingest-tools' import {Batch, DataSource} from '@subsquid/util-internal-processor-tools' import {getRequestAt, RangeRequest} from '@subsquid/util-internal-range' +import {Bytes32} from '@subsquid/evm-data' import {cast} from '@subsquid/util-internal-validation' import assert from 'assert' -import {Bytes32} from '../interfaces/base' import {FieldSelection} from '../interfaces/data' import {DataRequest} from '../interfaces/data-request' import { diff --git a/evm/evm-processor/src/ds-rpc/client.ts b/evm/evm-processor/src/ds-rpc/client.ts index 64d5092dc..dc91f49b8 100644 --- a/evm/evm-processor/src/ds-rpc/client.ts +++ b/evm/evm-processor/src/ds-rpc/client.ts @@ -23,13 +23,13 @@ import { } from '@subsquid/util-internal-range' import {BYTES, cast, object, SMALL_QTY} from '@subsquid/util-internal-validation' import {addTimeout, TimeoutError} from '@subsquid/util-timeout' +import {Bytes32} from '@subsquid/evm-data' +import {Rpc} from '@subsquid/evm-data/lib/rpc' import assert from 'assert' -import {Bytes32} from '../interfaces/base' import {DataRequest} from '../interfaces/data-request' import {Block} from '../mapping/entities' import {mapBlock} from './mapping' import {MappingRequest, toMappingRequest} from './request' -import {Rpc} from './rpc' const NO_REQUEST = toMappingRequest() diff --git a/evm/evm-processor/src/ds-rpc/mapping.ts b/evm/evm-processor/src/ds-rpc/mapping.ts index 323adfffe..64772779a 100644 --- a/evm/evm-processor/src/ds-rpc/mapping.ts +++ b/evm/evm-processor/src/ds-rpc/mapping.ts @@ -1,16 +1,17 @@ import {addErrorContext, assertNotNull, unexpectedCase} from '@subsquid/util-internal' import {cast, GetCastType} from '@subsquid/util-internal-validation' import {GetPropsCast} from '@subsquid/util-internal-validation/lib/composite/object' -import assert from 'assert' -import {Bytes, Bytes20, Bytes32} from '../interfaces/base' -import {FieldSelection} from '../interfaces/data' +import {Bytes, Bytes20, Bytes32} from '@subsquid/evm-data' +import {Block as RpcBlock, DebugStateDiffResult, DebugStateMap, TraceDiff, TraceStateDiff} from '@subsquid/evm-data/lib/rpc' import { EvmTraceCallAction, EvmTraceCallResult, EvmTraceCreateAction, EvmTraceCreateResult, EvmTraceSuicideAction -} from '../interfaces/evm' +} from '@subsquid/evm-data/lib/normalization' +import assert from 'assert' +import {FieldSelection} from '../interfaces/data' import { Block, BlockHeader, @@ -31,7 +32,6 @@ import {setUpRelations} from '../mapping/relations' import {getLogProps, getTraceFrameValidator, isEmpty} from '../mapping/schema' import {filterBlock} from './filter' import {MappingRequest} from './request' -import {Block as RpcBlock, DebugStateDiffResult, DebugStateMap, TraceDiff, TraceStateDiff} from './rpc-data' import {DebugFrame, getBlockValidator} from './schema' import {getTxHash} from './util' diff --git a/evm/evm-processor/src/ds-rpc/request.ts b/evm/evm-processor/src/ds-rpc/request.ts index 5b90752b6..dffca5132 100644 --- a/evm/evm-processor/src/ds-rpc/request.ts +++ b/evm/evm-processor/src/ds-rpc/request.ts @@ -1,7 +1,7 @@ +import {DataRequest as RpcDataRequest} from '@subsquid/evm-data/lib/rpc' +import {_EvmTx, _EvmTxReceipt} from '@subsquid/evm-data/lib/normalization' import {FieldSelection} from '../interfaces/data' import {DataRequest} from '../interfaces/data-request' -import {_EvmTx, _EvmTxReceipt} from '../interfaces/evm' -import {DataRequest as RpcDataRequest} from './rpc-data' export interface MappingRequest extends RpcDataRequest { diff --git a/evm/evm-processor/src/ds-rpc/schema.ts b/evm/evm-processor/src/ds-rpc/schema.ts index c2e49d812..1d2d129ea 100644 --- a/evm/evm-processor/src/ds-rpc/schema.ts +++ b/evm/evm-processor/src/ds-rpc/schema.ts @@ -14,7 +14,8 @@ import { Validator, withDefault } from '@subsquid/util-internal-validation' -import {Bytes, Bytes20} from '../interfaces/base' +import {DebugStateDiffResult, TraceStateDiff} from '@subsquid/evm-data/lib/rpc' +import {Bytes, Bytes20} from '@subsquid/evm-data' import {FieldSelection} from '../interfaces/data' import { getBlockHeaderProps, @@ -25,7 +26,6 @@ import { project } from '../mapping/schema' import {MappingRequest} from './request' -import {DebugStateDiffResult, TraceStateDiff} from './rpc-data' // Here we must be careful to include all fields, // that can potentially be used in item filters diff --git a/evm/evm-processor/src/ds-rpc/util.ts b/evm/evm-processor/src/ds-rpc/util.ts index e041ea38a..d4bd095c8 100644 --- a/evm/evm-processor/src/ds-rpc/util.ts +++ b/evm/evm-processor/src/ds-rpc/util.ts @@ -1,17 +1,4 @@ -import assert from 'assert' -import {Bytes32, Qty} from '../interfaces/base' - - -export function qty2Int(qty: Qty): number { - let i = parseInt(qty, 16) - assert(Number.isSafeInteger(i)) - return i -} - - -export function toQty(i: number): Qty { - return '0x' + i.toString(16) -} +import {Bytes32} from '@subsquid/evm-data' export function getTxHash(tx: Bytes32 | {hash: Bytes32}): Bytes32 { diff --git a/evm/evm-processor/src/interfaces/data-request.ts b/evm/evm-processor/src/interfaces/data-request.ts index a18404c19..5466798e5 100644 --- a/evm/evm-processor/src/interfaces/data-request.ts +++ b/evm/evm-processor/src/interfaces/data-request.ts @@ -1,6 +1,6 @@ -import {Bytes, Bytes20, Bytes32} from './base' +import {Bytes, Bytes20, Bytes32} from '@subsquid/evm-data' +import {EvmStateDiff} from '@subsquid/evm-data/lib/normalization' import {FieldSelection} from './data' -import {EvmStateDiff} from './evm' export interface DataRequest { diff --git a/evm/evm-processor/src/interfaces/data.ts b/evm/evm-processor/src/interfaces/data.ts index 4b5a6a6a1..1aaff214c 100644 --- a/evm/evm-processor/src/interfaces/data.ts +++ b/evm/evm-processor/src/interfaces/data.ts @@ -11,7 +11,7 @@ import { EvmTraceRewardAction, EvmTraceSuicideAction, EvmTransaction -} from './evm' +} from '@subsquid/evm-data/lib/normalization' type Simplify = { diff --git a/evm/evm-processor/src/interfaces/evm.ts b/evm/evm-processor/src/interfaces/evm.ts index b14872f66..8c7ab91e1 100644 --- a/evm/evm-processor/src/interfaces/evm.ts +++ b/evm/evm-processor/src/interfaces/evm.ts @@ -1,196 +1,196 @@ -import {Bytes, Bytes20, Bytes32, Bytes8} from './base' - - -export interface EvmBlockHeader { - height: number - hash: Bytes32 - parentHash: Bytes32 - nonce: Bytes8 - sha3Uncles: Bytes32 - logsBloom: Bytes - transactionsRoot: Bytes32 - stateRoot: Bytes32 - receiptsRoot: Bytes32 - mixHash: Bytes - miner: Bytes20 - difficulty: bigint - totalDifficulty: bigint - extraData: Bytes - size: bigint - gasLimit: bigint - gasUsed: bigint - timestamp: number - baseFeePerGas: bigint - /** - * This field is not supported by all currently deployed archives. - * Requesting it may cause internal error. - */ - l1BlockNumber: number -} - - -export interface EvmTransaction extends _EvmTx, _EvmTxReceipt { - transactionIndex: number - sighash: Bytes -} - - -export interface _EvmTx { - hash: Bytes32 - from: Bytes20 - to?: Bytes20 - gas: bigint - gasPrice: bigint - maxFeePerGas?: bigint - maxPriorityFeePerGas?: bigint - input: Bytes - nonce: number - value: bigint - v: bigint - r: Bytes32 - s: Bytes32 - yParity?: number - chainId?: number -} - - -export interface _EvmTxReceipt { - gasUsed: bigint - cumulativeGasUsed: bigint - effectiveGasPrice: bigint - contractAddress?: Bytes32 - type: number - status: number -} - - -export interface EvmLog { - logIndex: number - transactionIndex: number - transactionHash: Bytes32 - address: Bytes20 - data: Bytes - topics: Bytes32[] -} - - -export interface EvmTraceBase { - transactionIndex: number - traceAddress: number[] - subtraces: number - error: string | null - revertReason?: string -} - - -export interface EvmTraceCreate extends EvmTraceBase { - type: 'create' - action: EvmTraceCreateAction - result?: EvmTraceCreateResult -} - - -export interface EvmTraceCreateAction { - from: Bytes20 - value: bigint - gas: bigint - init: Bytes -} - - -export interface EvmTraceCreateResult { - gasUsed: bigint - code: Bytes - address: Bytes20 -} - - -export interface EvmTraceCall extends EvmTraceBase { - type: 'call' - action: EvmTraceCallAction - result?: EvmTraceCallResult -} - - -export interface EvmTraceCallAction { - callType: string - from: Bytes20 - to: Bytes20 - value?: bigint - gas: bigint - input: Bytes - sighash: Bytes -} +// import {Bytes, Bytes20, Bytes32, Bytes8} from '@subsquid/evm-data' + + +// export interface EvmBlockHeader { +// height: number +// hash: Bytes32 +// parentHash: Bytes32 +// nonce: Bytes8 +// sha3Uncles: Bytes32 +// logsBloom: Bytes +// transactionsRoot: Bytes32 +// stateRoot: Bytes32 +// receiptsRoot: Bytes32 +// mixHash: Bytes +// miner: Bytes20 +// difficulty: bigint +// totalDifficulty: bigint +// extraData: Bytes +// size: bigint +// gasLimit: bigint +// gasUsed: bigint +// timestamp: number +// baseFeePerGas: bigint +// /** +// * This field is not supported by all currently deployed archives. +// * Requesting it may cause internal error. +// */ +// l1BlockNumber: number +// } + + +// export interface EvmTransaction extends _EvmTx, _EvmTxReceipt { +// transactionIndex: number +// sighash: Bytes +// } + + +// export interface _EvmTx { +// hash: Bytes32 +// from: Bytes20 +// to?: Bytes20 +// gas: bigint +// gasPrice: bigint +// maxFeePerGas?: bigint +// maxPriorityFeePerGas?: bigint +// input: Bytes +// nonce: number +// value: bigint +// v: bigint +// r: Bytes32 +// s: Bytes32 +// yParity?: number +// chainId?: number +// } + + +// export interface _EvmTxReceipt { +// gasUsed: bigint +// cumulativeGasUsed: bigint +// effectiveGasPrice: bigint +// contractAddress?: Bytes32 +// type: number +// status: number +// } + + +// export interface EvmLog { +// logIndex: number +// transactionIndex: number +// transactionHash: Bytes32 +// address: Bytes20 +// data: Bytes +// topics: Bytes32[] +// } + + +// export interface EvmTraceBase { +// transactionIndex: number +// traceAddress: number[] +// subtraces: number +// error: string | null +// revertReason?: string +// } + + +// export interface EvmTraceCreate extends EvmTraceBase { +// type: 'create' +// action: EvmTraceCreateAction +// result?: EvmTraceCreateResult +// } + + +// export interface EvmTraceCreateAction { +// from: Bytes20 +// value: bigint +// gas: bigint +// init: Bytes +// } + + +// export interface EvmTraceCreateResult { +// gasUsed: bigint +// code: Bytes +// address: Bytes20 +// } + + +// export interface EvmTraceCall extends EvmTraceBase { +// type: 'call' +// action: EvmTraceCallAction +// result?: EvmTraceCallResult +// } + + +// export interface EvmTraceCallAction { +// callType: string +// from: Bytes20 +// to: Bytes20 +// value?: bigint +// gas: bigint +// input: Bytes +// sighash: Bytes +// } -export interface EvmTraceCallResult { - gasUsed: bigint - output: Bytes -} +// export interface EvmTraceCallResult { +// gasUsed: bigint +// output: Bytes +// } -export interface EvmTraceSuicide extends EvmTraceBase { - type: 'suicide' - action: EvmTraceSuicideAction -} +// export interface EvmTraceSuicide extends EvmTraceBase { +// type: 'suicide' +// action: EvmTraceSuicideAction +// } -export interface EvmTraceSuicideAction { - address: Bytes20 - refundAddress: Bytes20 - balance: bigint -} +// export interface EvmTraceSuicideAction { +// address: Bytes20 +// refundAddress: Bytes20 +// balance: bigint +// } -export interface EvmTraceReward extends EvmTraceBase { - type: 'reward' - action: EvmTraceRewardAction -} +// export interface EvmTraceReward extends EvmTraceBase { +// type: 'reward' +// action: EvmTraceRewardAction +// } -export interface EvmTraceRewardAction { - author: Bytes20 - value: bigint - type: string -} +// export interface EvmTraceRewardAction { +// author: Bytes20 +// value: bigint +// type: string +// } -export type EvmTrace = EvmTraceCreate | EvmTraceCall | EvmTraceSuicide | EvmTraceReward +// export type EvmTrace = EvmTraceCreate | EvmTraceCall | EvmTraceSuicide | EvmTraceReward -export interface EvmStateDiffBase { - transactionIndex: number - address: Bytes20 - key: 'balance' | 'code' | 'nonce' | Bytes32 -} +// export interface EvmStateDiffBase { +// transactionIndex: number +// address: Bytes20 +// key: 'balance' | 'code' | 'nonce' | Bytes32 +// } -export interface EvmStateDiffNoChange extends EvmStateDiffBase { - kind: '=' - prev?: null - next?: null -} +// export interface EvmStateDiffNoChange extends EvmStateDiffBase { +// kind: '=' +// prev?: null +// next?: null +// } -export interface EvmStateDiffAdd extends EvmStateDiffBase { - kind: '+' - prev?: null - next: Bytes -} +// export interface EvmStateDiffAdd extends EvmStateDiffBase { +// kind: '+' +// prev?: null +// next: Bytes +// } -export interface EvmStateDiffChange extends EvmStateDiffBase { - kind: '*' - prev: Bytes - next: Bytes -} +// export interface EvmStateDiffChange extends EvmStateDiffBase { +// kind: '*' +// prev: Bytes +// next: Bytes +// } -export interface EvmStateDiffDelete extends EvmStateDiffBase { - kind: '-' - prev: Bytes - next?: null -} +// export interface EvmStateDiffDelete extends EvmStateDiffBase { +// kind: '-' +// prev: Bytes +// next?: null +// } -export type EvmStateDiff = EvmStateDiffNoChange | EvmStateDiffAdd | EvmStateDiffChange | EvmStateDiffDelete +// export type EvmStateDiff = EvmStateDiffNoChange | EvmStateDiffAdd | EvmStateDiffChange | EvmStateDiffDelete diff --git a/evm/evm-processor/src/mapping/entities.ts b/evm/evm-processor/src/mapping/entities.ts index 1badef178..f6e9ea259 100644 --- a/evm/evm-processor/src/mapping/entities.ts +++ b/evm/evm-processor/src/mapping/entities.ts @@ -1,6 +1,5 @@ import {formatId} from '@subsquid/util-internal-processor-tools' -import assert from 'assert' -import {Bytes, Bytes20, Bytes32, Bytes8} from '../interfaces/base' +import {Bytes, Bytes20, Bytes32, Bytes8} from '@subsquid/evm-data' import { EvmTraceCallAction, EvmTraceCallResult, @@ -8,7 +7,8 @@ import { EvmTraceCreateResult, EvmTraceRewardAction, EvmTraceSuicideAction -} from '../interfaces/evm' +} from '@subsquid/evm-data/lib/normalization' +import assert from 'assert' export class Block { diff --git a/evm/evm-processor/src/mapping/schema.ts b/evm/evm-processor/src/mapping/schema.ts index b17f2a60b..993157764 100644 --- a/evm/evm-processor/src/mapping/schema.ts +++ b/evm/evm-processor/src/mapping/schema.ts @@ -71,12 +71,12 @@ export function getTxProps(fields: FieldSelection['transaction'], forArchive: bo export function getTxReceiptProps(fields: FieldSelection['transaction'], forArchive: boolean) { let natural = forArchive ? NAT : SMALL_QTY return project(fields, { - gasUsed: withSentinel('Receipt.gasUsed', -1n, QTY), - cumulativeGasUsed: withSentinel('Receipt.cumulativeGasUsed', -1n, QTY), - effectiveGasPrice: withSentinel('Receipt.effectiveGasPrice', -1n, QTY), - contractAddress: option(BYTES), - type: withSentinel('Receipt.type', -1, natural), - status: withSentinel('Receipt.status', -1, natural), + // gasUsed: withSentinel('Receipt.gasUsed', -1n, QTY), + // cumulativeGasUsed: withSentinel('Receipt.cumulativeGasUsed', -1n, QTY), + // effectiveGasPrice: withSentinel('Receipt.effectiveGasPrice', -1n, QTY), + // contractAddress: option(BYTES), + // type: withSentinel('Receipt.type', -1, natural), + // status: withSentinel('Receipt.status', -1, natural), }) } diff --git a/evm/evm-processor/src/mapping/selection.ts b/evm/evm-processor/src/mapping/selection.ts index 548cc9f81..a3005ec38 100644 --- a/evm/evm-processor/src/mapping/selection.ts +++ b/evm/evm-processor/src/mapping/selection.ts @@ -50,12 +50,12 @@ export function getTxSelectionValidator() { s: FIELD, yParity: FIELD, chainId: FIELD, - gasUsed: FIELD, - cumulativeGasUsed: FIELD, - effectiveGasPrice: FIELD, - contractAddress: FIELD, - type: FIELD, - status: FIELD, + // gasUsed: FIELD, + // cumulativeGasUsed: FIELD, + // effectiveGasPrice: FIELD, + // contractAddress: FIELD, + // type: FIELD, + // status: FIELD, } return object(fields) } diff --git a/rush.json b/rush.json index dfacf1dfb..82d93d400 100644 --- a/rush.json +++ b/rush.json @@ -450,12 +450,24 @@ // */ // // "versionPolicyName": "" // }, + { + "packageName": "@subsquid/evm-data", + "projectFolder": "evm/evm-data", + "shouldPublish": true, + "versionPolicyName": "npm" + }, { "packageName": "@subsquid/evm-dump", "projectFolder": "evm/evm-dump", "shouldPublish": true, "versionPolicyName": "npm" }, + { + "packageName": "@subsquid/evm-ingest", + "projectFolder": "evm/evm-ingest", + "shouldPublish": true, + "versionPolicyName": "npm" + }, { "packageName": "@subsquid/evm-processor", "projectFolder": "evm/evm-processor", From 456ca72971678835a614ae15840b4167e5c3c181 Mon Sep 17 00:00:00 2001 From: tmcgroul Date: Sat, 9 Mar 2024 18:41:34 +0700 Subject: [PATCH 3/5] try to reuse already defined schema --- evm/evm-data/src/rpc/rpc-data.ts | 6 +++--- evm/evm-processor/src/ds-rpc/schema.ts | 10 ++-------- util/util-internal-validation/src/dsl.ts | 6 +++--- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/evm/evm-data/src/rpc/rpc-data.ts b/evm/evm-data/src/rpc/rpc-data.ts index 8641f8618..61907cf8f 100644 --- a/evm/evm-data/src/rpc/rpc-data.ts +++ b/evm/evm-data/src/rpc/rpc-data.ts @@ -60,7 +60,7 @@ export interface Block { } -const Transaction = object({ +export const TransactionSchema = object({ blockNumber: SMALL_QTY, blockHash: BYTES, transactionIndex: SMALL_QTY, @@ -83,7 +83,7 @@ const Transaction = object({ }) -export type Transaction = GetSrcType +export type Transaction = GetSrcType const GetBlockBase = { @@ -112,7 +112,7 @@ const GetBlockBase = { export const GetBlockWithTransactions = object({ ...GetBlockBase, - transactions: array(Transaction) + transactions: array(TransactionSchema) }) diff --git a/evm/evm-processor/src/ds-rpc/schema.ts b/evm/evm-processor/src/ds-rpc/schema.ts index 1d2d129ea..de117346a 100644 --- a/evm/evm-processor/src/ds-rpc/schema.ts +++ b/evm/evm-processor/src/ds-rpc/schema.ts @@ -14,7 +14,7 @@ import { Validator, withDefault } from '@subsquid/util-internal-validation' -import {DebugStateDiffResult, TraceStateDiff} from '@subsquid/evm-data/lib/rpc' +import {DebugStateDiffResult, TraceStateDiff, TransactionSchema} from '@subsquid/evm-data/lib/rpc' import {Bytes, Bytes20} from '@subsquid/evm-data' import {FieldSelection} from '../interfaces/data' import { @@ -32,13 +32,7 @@ import {MappingRequest} from './request' // (no matter what field selection is telling us to omit) export const getBlockValidator = weakMemo((req: MappingRequest) => { let Transaction = req.transactions - ? object({ - ...getTxProps(req.fields.transaction, false), - hash: BYTES, - input: BYTES, - from: BYTES, - to: option(BYTES), - }) + ? object(project(req.fields.transaction, TransactionSchema.props)) : BYTES let GetBlock = object({ diff --git a/util/util-internal-validation/src/dsl.ts b/util/util-internal-validation/src/dsl.ts index 83950e115..39b552fc0 100644 --- a/util/util-internal-validation/src/dsl.ts +++ b/util/util-internal-validation/src/dsl.ts @@ -15,9 +15,9 @@ import {DataValidationError, ValidationFailure} from './error' import {GetCastType, GetSrcType, Validator} from './interface' -export function object | undefined>>( +export function object>>( props: Props -): Validator, GetPropsSrc> { +): ObjectValidator { let presentProps: Record> = {} for (let key in props) { let v = props[key] @@ -25,7 +25,7 @@ export function object | undefined>> presentProps[key] = v } } - return new ObjectValidator(presentProps) as any + return new ObjectValidator(presentProps) as ObjectValidator } From 563f4433755be4fbcdfa21070398f5e5966e163b Mon Sep 17 00:00:00 2001 From: tmcgroul Date: Sat, 16 Mar 2024 13:24:55 +0700 Subject: [PATCH 4/5] commit changes --- evm/evm-data/src/normalization/data.ts | 110 +++--- evm/evm-data/src/normalization/mapping.ts | 352 ++++++++++++++---- evm/evm-data/src/rpc/rpc-data.ts | 68 +--- evm/evm-data/src/schema.ts | 303 +++++++++++++++ evm/evm-dump/package.json | 1 + evm/evm-dump/src/dumper.ts | 22 +- evm/evm-ingest/src/ingest.ts | 2 +- evm/evm-processor/src/ds-archive/schema.ts | 52 ++- evm/evm-processor/src/ds-rpc/mapping.ts | 9 +- evm/evm-processor/src/ds-rpc/request.ts | 2 +- evm/evm-processor/src/ds-rpc/schema.ts | 33 +- .../src/interfaces/data-request.ts | 2 +- evm/evm-processor/src/interfaces/data.ts | 2 +- evm/evm-processor/src/interfaces/evm.ts | 344 ++++++++--------- evm/evm-processor/src/mapping/entities.ts | 4 +- evm/evm-processor/src/mapping/schema.ts | 91 +---- evm/evm-processor/src/mapping/selection.ts | 12 +- .../src/composite/object.ts | 6 +- util/util-internal-validation/src/dsl.ts | 11 +- 19 files changed, 928 insertions(+), 498 deletions(-) create mode 100644 evm/evm-data/src/schema.ts diff --git a/evm/evm-data/src/normalization/data.ts b/evm/evm-data/src/normalization/data.ts index fd86d4748..71070ba64 100644 --- a/evm/evm-data/src/normalization/data.ts +++ b/evm/evm-data/src/normalization/data.ts @@ -1,25 +1,25 @@ import {Bytes, Bytes20, Bytes32, Bytes8} from '../base' -export interface EvmBlockHeader { +export interface BlockHeader { height: number hash: Bytes32 parentHash: Bytes32 nonce?: Bytes8 - sha3Uncles: Bytes32 - logsBloom: Bytes - transactionsRoot: Bytes32 - stateRoot: Bytes32 - receiptsRoot: Bytes32 + sha3Uncles?: Bytes32 + logsBloom?: Bytes + transactionsRoot?: Bytes32 + stateRoot?: Bytes32 + receiptsRoot?: Bytes32 mixHash?: Bytes - miner: Bytes20 + miner?: Bytes20 difficulty?: bigint totalDifficulty?: bigint - extraData: Bytes - size: bigint - gasLimit: bigint - gasUsed: bigint - timestamp: number + extraData?: Bytes + size?: bigint + gasLimit?: bigint + gasUsed?: bigint + timestamp?: number baseFeePerGas?: bigint /** * This field is not supported by all currently deployed archives. @@ -29,23 +29,23 @@ export interface EvmBlockHeader { } -export interface EvmTransaction extends _EvmTx, _EvmTxReceipt { +export interface Transaction extends _Tx, _TxReceipt { transactionIndex: number sighash: Bytes } -export interface _EvmTx { +export interface _Tx { hash: Bytes32 from: Bytes20 to?: Bytes20 - gas: bigint - gasPrice: bigint + gas?: bigint + gasPrice?: bigint maxFeePerGas?: bigint maxPriorityFeePerGas?: bigint input: Bytes - nonce: number - value: bigint + nonce?: number + value?: bigint v?: bigint r?: Bytes32 s?: Bytes32 @@ -54,17 +54,17 @@ export interface _EvmTx { } -export interface _EvmTxReceipt { - // gasUsed: bigint - // cumulativeGasUsed: bigint - // effectiveGasPrice: bigint - // contractAddress?: Bytes32 - // type: number - // status: number +export interface _TxReceipt { + gasUsed?: bigint + cumulativeGasUsed?: bigint + effectiveGasPrice?: bigint + contractAddress?: Bytes32 + type?: number + status?: number } -export interface EvmLog { +export interface Log { logIndex: number transactionIndex: number transactionHash: Bytes32 @@ -74,7 +74,7 @@ export interface EvmLog { } -export interface EvmTraceBase { +export interface TraceBase { transactionIndex: number traceAddress: number[] subtraces: number @@ -83,14 +83,14 @@ export interface EvmTraceBase { } -export interface EvmTraceCreate extends EvmTraceBase { +export interface TraceCreate extends TraceBase { type: 'create' - action: EvmTraceCreateAction - result?: EvmTraceCreateResult + action: TraceCreateAction + result?: TraceCreateResult } -export interface EvmTraceCreateAction { +export interface TraceCreateAction { from: Bytes20 value: bigint gas: bigint @@ -98,21 +98,21 @@ export interface EvmTraceCreateAction { } -export interface EvmTraceCreateResult { +export interface TraceCreateResult { gasUsed: bigint code: Bytes address: Bytes20 } -export interface EvmTraceCall extends EvmTraceBase { +export interface TraceCall extends TraceBase { type: 'call' - action: EvmTraceCallAction - result?: EvmTraceCallResult + action: TraceCallAction + result?: TraceCallResult } -export interface EvmTraceCallAction { +export interface TraceCallAction { callType: string from: Bytes20 to: Bytes20 @@ -123,83 +123,83 @@ export interface EvmTraceCallAction { } -export interface EvmTraceCallResult { +export interface TraceCallResult { gasUsed: bigint output: Bytes } -export interface EvmTraceSuicide extends EvmTraceBase { +export interface TraceSuicide extends TraceBase { type: 'suicide' - action: EvmTraceSuicideAction + action: TraceSuicideAction } -export interface EvmTraceSuicideAction { +export interface TraceSuicideAction { address: Bytes20 refundAddress: Bytes20 balance: bigint } -export interface EvmTraceReward extends EvmTraceBase { +export interface TraceReward extends TraceBase { type: 'reward' - action: EvmTraceRewardAction + action: TraceRewardAction } -export interface EvmTraceRewardAction { +export interface TraceRewardAction { author: Bytes20 value: bigint type: string } -export type EvmTrace = EvmTraceCreate | EvmTraceCall | EvmTraceSuicide | EvmTraceReward +export type Trace = TraceCreate | TraceCall | TraceSuicide | TraceReward -export interface EvmStateDiffBase { +export interface StateDiffBase { transactionIndex: number address: Bytes20 key: 'balance' | 'code' | 'nonce' | Bytes32 } -export interface EvmStateDiffNoChange extends EvmStateDiffBase { +export interface StateDiffNoChange extends StateDiffBase { kind: '=' prev?: null next?: null } -export interface EvmStateDiffAdd extends EvmStateDiffBase { +export interface StateDiffAdd extends StateDiffBase { kind: '+' prev?: null next: Bytes } -export interface EvmStateDiffChange extends EvmStateDiffBase { +export interface StateDiffChange extends StateDiffBase { kind: '*' prev: Bytes next: Bytes } -export interface EvmStateDiffDelete extends EvmStateDiffBase { +export interface EvmStateDiffDelete extends StateDiffBase { kind: '-' prev: Bytes next?: null } -export type EvmStateDiff = EvmStateDiffNoChange | EvmStateDiffAdd | EvmStateDiffChange | EvmStateDiffDelete +export type StateDiff = StateDiffNoChange | StateDiffAdd | StateDiffChange | EvmStateDiffDelete export interface Block { - header: EvmBlockHeader - transactions: EvmTransaction[] - logs: EvmLog[] - traces: EvmTrace[] - stateDiffs: EvmStateDiff[] + header: BlockHeader + transactions: Transaction[] + logs: Log[] + traces: Trace[] + stateDiffs: StateDiff[] } diff --git a/evm/evm-data/src/normalization/mapping.ts b/evm/evm-data/src/normalization/mapping.ts index c62739b9b..b2a8ba5bf 100644 --- a/evm/evm-data/src/normalization/mapping.ts +++ b/evm/evm-data/src/normalization/mapping.ts @@ -1,57 +1,71 @@ -import {addErrorContext, groupBy, unexpectedCase} from '@subsquid/util-internal' +import {assertNotNull, unexpectedCase} from '@subsquid/util-internal' import assert from 'assert' -import * as rpc from '../rpc' -import {Block, EvmBlockHeader, EvmTransaction, EvmLog, EvmTrace, EvmStateDiff} from './data' +import * as rpc from '../schema' +import {Block, BlockHeader, Transaction, Log, Trace, StateDiff, TraceSuicide, TraceCreate, TraceCall, TraceReward} from './data' +import {Bytes20} from '../base' export function mapRpcBlock(src: rpc.Block): Block { - let header: EvmBlockHeader = { - hash: src.hash, - height: src.height, - parentHash: src.block.parentHash, - logsBloom: src.block.logsBloom, - extraData: src.block.extraData, - gasLimit: BigInt(src.block.gasLimit), - gasUsed: BigInt(src.block.gasUsed), - miner: src.block.miner, - receiptsRoot: src.block.receiptsRoot, - sha3Uncles: src.block.sha3Uncles, - size: BigInt(src.block.size), - stateRoot: src.block.stateRoot, - timestamp: parseInt(src.block.timestamp), - transactionsRoot: src.block.transactionsRoot, - } + let header = mapBlockHeader(src.block) + let transactions: Transaction[] = [] + let logs: Log[] = [] + let traces: Trace[] = [] + let stateDiffs: StateDiff[] = [] - if (src.block.baseFeePerGas) { - header.baseFeePerGas = BigInt(src.block.baseFeePerGas) - } - if (src.block.difficulty) { - header.difficulty = BigInt(src.block.difficulty) - } - if (src.block.totalDifficulty) { - header.totalDifficulty = BigInt(src.block.totalDifficulty) - } - if (src.block.l1BlockNumber) { - header.l1BlockNumber = parseInt(src.block.l1BlockNumber) - } - if (src.block.mixHash) { - header.mixHash = src.block.mixHash + let txByHash: Record = {} + for (let i = 0; i < src.block.transactions.length; i++) { + let tx = src.block.transactions[i] + if (typeof tx == 'string') continue + + txByHash[tx.hash] = tx + + let receipt = src.receipts?.[i] + if (receipt) { + assert(receipt.transactionIndex == tx.transactionIndex) + } + + let transaction = mapTransaction(tx, receipt) + transactions.push(transaction) + + for (let log of receipt?.logs || []) { + logs.push(mapLog(log)) + } } - if (src.block.nonce) { - header.nonce = src.block.nonce + + if (src.logs) { + assert(logs.length == 0) + for (let log of src.logs) { + logs.push(mapLog(log)) + } } - let transactions: EvmTransaction[] = [] - let logs: EvmLog[] = [] - let traces: EvmTrace[] = [] - let stateDiffs: EvmStateDiff[] = [] + for (let replay of src.traceReplays || []) { + let transactionHash = assertNotNull(replay.transactionHash) + let transactionIndex = parseInt(txByHash[transactionHash].transactionIndex) - for (let tx of src.block.transactions) { - if (typeof tx == 'string') continue - let transaction = mapRpcTransaction(tx) - transactions.push(transaction) + for (let trace of replay.trace || []) { + let transactionHash = assertNotNull(replay.transactionHash || trace.transactionHash) + let transactionIndex = parseInt(txByHash[transactionHash].transactionIndex) + traces.push(mapReplayTrace(trace, transactionIndex)) + } + + if (replay.stateDiff) { + for (let diff of mapReplayStateDiff(replay.stateDiff, transactionIndex)) { + if (diff.kind != '=') { + stateDiffs.push(diff) + } + } + } } + // for (let frame of src.debugFrames || []) { + // traces.push(mapDebugFrame(frame)) + // } + + // for (let result of src.debugStateDiffs || []) { + // stateDiffs.push(mapDebugStateDiff(result)) + // } + return { header, transactions, @@ -62,42 +76,224 @@ export function mapRpcBlock(src: rpc.Block): Block { } -function mapRpcTransaction(src: rpc.Transaction): EvmTransaction { - let tx: EvmTransaction = { +function* mapReplayStateDiff( + src: Record, + transactionIndex: number, +): Iterable { + for (let address in src) { + let diffs = src[address] + yield makeStateDiffFromReplay(transactionIndex, address, 'code', diffs.code) + yield makeStateDiffFromReplay(transactionIndex, address, 'balance', diffs.balance) + yield makeStateDiffFromReplay(transactionIndex, address, 'nonce', diffs.nonce) + for (let key in diffs.storage) { + yield makeStateDiffFromReplay(transactionIndex, address, key, diffs.storage[key]) + } + } +} + + +function makeStateDiffFromReplay( + transactionIndex: number, + address: Bytes20, + key: string, + diff: rpc.TraceDiff +): StateDiff { + let base = { + address, + key, + transactionIndex + } + if (diff === '=') { + return { + ...base, + kind: '=' + } + } + if (diff['+']) { + return { + ...base, + kind: '+', + next: diff['+'] + } + } + if (diff['*']) { + return { + ...base, + kind: '*', + prev: diff['*'].from, + next: diff['*'].to, + } + } + if (diff['-']) { + return { + ...base, + kind: '-', + prev: diff['-'] + } + } + throw unexpectedCase() +} + + +function mapReplayTrace(src: rpc.TraceFrame, transactionIndex: number): Trace { + let base = { + transactionIndex, + traceAddress: src.traceAddress, + subtraces: src.subtraces, + error: src.error ?? null, + } + + switch (src.type) { + case 'create': { + let trace: TraceCreate = { + ...base, + type: src.type, + action: { + from: src.action.from, + value: BigInt(src.action.value), + gas: BigInt(src.action.gas), + init: src.action.init + } + } + if (src.result) { + trace.result = { + gasUsed: BigInt(src.result.gasUsed), + code: src.result.code, + address: src.result.address, + } + } + return trace + } + case 'call': { + let trace: TraceCall = { + ...base, + type: src.type, + action: { + from: src.action.from, + to: src.action.to, + value: BigInt(src.action.value), + gas: BigInt(src.action.gas), + input: src.action.input, + sighash: src.action.input.slice(0, 10), + callType: src.action.callType + } + } + if (src.result) { + trace.result = { + gasUsed: BigInt(src.result?.gasUsed), + output: src.result.output + } + } + return trace + } + case 'reward': { + let trace: TraceReward = { + ...base, + type: src.type, + action: { + author: src.action.author, + value: BigInt(src.action.value), + type: src.action.rewardType + }, + } + return trace + } + case 'suicide': { + let trace: TraceSuicide = { + ...base, + type: src.type, + action: { + address: src.action.address, + refundAddress: src.action.refundAddress, + balance: BigInt(src.action.balance) + } + } + return trace + } + default: + throw unexpectedCase() + } +} + + +function mapBlockHeader(src: rpc.GetBlock) { + let header: BlockHeader = { + hash: src.hash, + height: parseInt(src.number), + parentHash: src.parentHash, + logsBloom: src.logsBloom ?? undefined, + extraData: src.extraData ?? undefined, + miner: src.miner ?? undefined, + receiptsRoot: src.receiptsRoot ?? undefined, + sha3Uncles: src.sha3Uncles ?? undefined, + stateRoot: src.stateRoot ?? undefined, + transactionsRoot: src.transactionsRoot ?? undefined, + mixHash: src.mixHash ?? undefined, + nonce: src.nonce ?? undefined, + } + + if (src.gasLimit) { + header.gasLimit = BigInt(src.gasLimit) + } + if (src.gasUsed) { + header.gasUsed = BigInt(src.gasUsed) + } + if (src.size) { + header.size = BigInt(src.size) + } + if (src.timestamp) { + header.timestamp = parseInt(src.timestamp) + } + if (src.baseFeePerGas) { + header.baseFeePerGas = BigInt(src.baseFeePerGas) + } + if (src.difficulty) { + header.difficulty = BigInt(src.difficulty) + } + if (src.totalDifficulty) { + header.totalDifficulty = BigInt(src.totalDifficulty) + } + if (src.l1BlockNumber) { + header.l1BlockNumber = parseInt(src.l1BlockNumber) + } + + return header +} + + +function mapTransaction(src: rpc.Transaction, receipt?: rpc.TransactionReceipt): Transaction { + let tx: Transaction = { transactionIndex: parseInt(src.transactionIndex), sighash: src.input.slice(0, 10), hash: src.hash, from: src.from, - gas: BigInt(src.gas), - gasPrice: BigInt(src.gasPrice), input: src.input, - nonce: parseInt(src.nonce), - value: BigInt(src.value), - // gasUsed: bigint - // cumulativeGasUsed: bigint - // effectiveGasPrice: bigint - // contractAddress?: Bytes32 - // type: number - // status: number + to: src.to ?? undefined, + r: src.r ?? undefined, + s: src.s ?? undefined, + contractAddress: receipt?.contractAddress ?? undefined } - if (src.to) { - tx.to = src.to - } if (src.maxFeePerGas) { tx.maxFeePerGas = BigInt(src.maxFeePerGas) } if (src.maxPriorityFeePerGas) { tx.maxPriorityFeePerGas = BigInt(src.maxPriorityFeePerGas) } - if (src.v) { - tx.v = BigInt(src.v) + if (src.gas) { + tx.gas = BigInt(src.gas) + } + if (src.gasPrice) { + tx.gasPrice = BigInt(src.gasPrice) + } + if (src.nonce) { + tx.nonce = parseInt(src.nonce) } - if (src.r) { - tx.r = src.r + if (src.value) { + tx.value = BigInt(src.value) } - if (src.s) { - tx.s = src.s + if (src.v) { + tx.v = BigInt(src.v) } if (src.yParity) { tx.yParity = parseInt(src.yParity) @@ -106,5 +302,33 @@ function mapRpcTransaction(src: rpc.Transaction): EvmTransaction { tx.chainId = parseInt(src.chainId) } + if (receipt?.gasUsed) { + tx.gasUsed = BigInt(receipt.gasUsed) + } + if (receipt?.cumulativeGasUsed) { + tx.cumulativeGasUsed = BigInt(receipt.cumulativeGasUsed) + } + if (receipt?.effectiveGasPrice) { + tx.effectiveGasPrice = BigInt(receipt.effectiveGasPrice) + } + if (receipt?.type) { + tx.type = parseInt(receipt.type) + } + if (receipt?.status) { + tx.status = parseInt(receipt.status) + } + return tx } + + +function mapLog(src: rpc.Log): Log { + return { + logIndex: parseInt(src.logIndex), + transactionIndex: parseInt(src.transactionIndex), + transactionHash: src.transactionHash, + address: src.address, + data: src.data, + topics: src.topics, + } +} diff --git a/evm/evm-data/src/rpc/rpc-data.ts b/evm/evm-data/src/rpc/rpc-data.ts index 61907cf8f..95441381b 100644 --- a/evm/evm-data/src/rpc/rpc-data.ts +++ b/evm/evm-data/src/rpc/rpc-data.ts @@ -15,10 +15,10 @@ import { STRING, Validator } from '@subsquid/util-internal-validation' -import {Bytes, Bytes8, Bytes20, Bytes32, Qty} from '../base' +import {Bytes, Bytes20, Bytes32, Qty} from '../base' -export function project( +function project( fields: F | undefined, obj: T ): Partial { @@ -60,64 +60,32 @@ export interface Block { } -export const TransactionSchema = object({ +export const Transaction = object({ blockNumber: SMALL_QTY, blockHash: BYTES, transactionIndex: SMALL_QTY, hash: BYTES, input: BYTES, - nonce: SMALL_QTY, - from: BYTES, - to: option(BYTES), - value: QTY, - type: option(SMALL_QTY), - gas: QTY, - gasPrice: QTY, - maxFeePerGas: option(QTY), - maxPriorityFeePerGas: option(QTY), - v: option(QTY), - r: option(BYTES), - s: option(BYTES), - yParity: option(SMALL_QTY), - chainId: option(SMALL_QTY) }) -export type Transaction = GetSrcType +export type Transaction = GetSrcType -const GetBlockBase = { +export const GetBlockWithTransactions = object({ number: SMALL_QTY, hash: BYTES, parentHash: BYTES, logsBloom: BYTES, - timestamp: SMALL_QTY, - transactionsRoot: BYTES, - receiptsRoot: BYTES, - stateRoot: BYTES, - sha3Uncles: BYTES, - extraData: BYTES, - miner: BYTES, - nonce: option(BYTES), - mixHash: option(BYTES), - size: SMALL_QTY, - gasLimit: QTY, - gasUsed: QTY, - difficulty: option(QTY), - totalDifficulty: option(QTY), - baseFeePerGas: option(QTY), - l1BlockNumber: option(SMALL_QTY) -} - - -export const GetBlockWithTransactions = object({ - ...GetBlockBase, - transactions: array(TransactionSchema) + transactions: array(Transaction) }) export const GetBlockNoTransactions = object({ - ...GetBlockBase, + number: SMALL_QTY, + hash: BYTES, + parentHash: BYTES, + logsBloom: BYTES, transactions: array(BYTES) }) @@ -127,22 +95,6 @@ export interface GetBlock { hash: Bytes32 parentHash: Bytes32 logsBloom: Bytes - timestamp: Qty - transactionsRoot: Bytes32 - receiptsRoot: Bytes32 - stateRoot: Bytes32 - sha3Uncles: Bytes32 - extraData: Bytes - miner: Bytes20 - nonce?: Bytes8 | null - mixHash?: Bytes | null - size: Qty - gasLimit: Qty - gasUsed: Qty - difficulty?: Qty | null - totalDifficulty?: Qty | null - baseFeePerGas?: Qty | null - l1BlockNumber?: Qty | null transactions: Bytes32[] | Transaction[] } diff --git a/evm/evm-data/src/schema.ts b/evm/evm-data/src/schema.ts new file mode 100644 index 000000000..3b3c74e75 --- /dev/null +++ b/evm/evm-data/src/schema.ts @@ -0,0 +1,303 @@ +import { + array, + BYTES, + constant, + GetSrcType, + keyTaggedUnion, + NAT, + object, + oneOf, + option, + QTY, + record, + ref, + SMALL_QTY, + STRING, + Validator, + withSentinel, + withDefault, + taggedUnion +} from '@subsquid/util-internal-validation' +import {Bytes} from './base' + + +export const Transaction = object({ + blockNumber: SMALL_QTY, + blockHash: BYTES, + transactionIndex: SMALL_QTY, + hash: BYTES, + input: BYTES, + nonce: withSentinel('Transaction.nonce', -1, SMALL_QTY), + from: BYTES, + to: option(BYTES), + value: withSentinel('Transaction.value', -1n, QTY), + gas: withSentinel('Transaction.gas', -1n, QTY), + gasPrice: withSentinel('Transaction.gasPrice', -1n, QTY), + maxFeePerGas: option(QTY), + maxPriorityFeePerGas: option(QTY), + v: withSentinel('Transaction.v', -1n, QTY), + r: withSentinel('Transaction.r', '0x', BYTES), + s: withSentinel('Transaction.s', '0x', BYTES), + yParity: option(SMALL_QTY), + chainId: option(SMALL_QTY) +}) + + +export type Transaction = GetSrcType + + +export const GetBlock = object({ + number: SMALL_QTY, + hash: BYTES, + parentHash: BYTES, + logsBloom: withSentinel('BlockHeader.logsBloom', '0x', BYTES), + timestamp: withSentinel('BlockHeader.timestamp', 0, SMALL_QTY), + transactionsRoot: withSentinel('BlockHeader.transactionsRoot', '0x', BYTES), + receiptsRoot: withSentinel('BlockHeader.receiptsRoot', '0x', BYTES), + stateRoot: withSentinel('BlockHeader.stateRoot', '0x', BYTES), + sha3Uncles: withSentinel('BlockHeader.sha3Uncles', '0x', BYTES), + extraData: withSentinel('BlockHeader.extraData', '0x', BYTES), + miner: withSentinel('BlockHeader.miner', '0x', BYTES), + nonce: withSentinel('BlockHeader.nonce', '0x', BYTES), + mixHash: withSentinel('BlockHeader.mixHash', '0x', BYTES), + size: withSentinel('BlockHeader.size', -1, SMALL_QTY), + gasLimit: withSentinel('BlockHeader.gasLimit', -1n, QTY), + gasUsed: withSentinel('BlockHeader.gasUsed', -1n, QTY), + difficulty: withSentinel('BlockHeader.difficulty', -1n, QTY), + totalDifficulty: withSentinel('BlockHeader.totalDifficulty', -1n, QTY), + baseFeePerGas: withSentinel('BlockHeader.baseFeePerGas', -1n, QTY), + l1BlockNumber: withDefault(0, SMALL_QTY), + transactions: oneOf({ + withTransactions: array(Transaction), + noTransactions: array(BYTES), + }) +}) + + +export type GetBlock = GetSrcType + + +export const Log = object({ + blockNumber: SMALL_QTY, + blockHash: BYTES, + logIndex: SMALL_QTY, + transactionIndex: SMALL_QTY, + transactionHash: BYTES, + address: BYTES, + data: BYTES, + topics: array(BYTES) +}) + + +export type Log = GetSrcType + + +export const TransactionReceipt = object({ + blockNumber: SMALL_QTY, + blockHash: BYTES, + transactionIndex: SMALL_QTY, + transactionHash: BYTES, + cumulativeGasUsed: withSentinel('Receipt.cumulativeGasUsed', -1n, QTY), + effectiveGasPrice: withSentinel('Receipt.effectiveGasPrice', -1n, QTY), + gasUsed: withSentinel('Receipt.gasUsed', -1n, QTY), + contractAddress: option(BYTES), + type: withSentinel('Receipt.type', -1, SMALL_QTY), + status: withSentinel('Receipt.status', -1, SMALL_QTY), + logs: array(Log) +}) + + +export type TransactionReceipt = GetSrcType + + +export const DebugFrame: Validator = object({ + type: STRING, + input: BYTES, + calls: option(array(ref(() => DebugFrame))) +}) + + +export interface DebugFrame { + type: string + input: Bytes + calls?: DebugFrame[] | null +} + + +export const DebugFrameResult = object({ + result: DebugFrame, + txHash: option(BYTES) +}) + + +export type DebugFrameResult = GetSrcType + + +export const DebugStateMap = object({ + balance: option(QTY), + code: option(BYTES), + nonce: option(SMALL_QTY), + storage: option(record(BYTES, BYTES)) +}) + + +export type DebugStateMap = GetSrcType + + +export const DebugStateDiff = object({ + pre: record(BYTES, DebugStateMap), + post: record(BYTES, DebugStateMap) +}) + + +export type DebugStateDiff = GetSrcType + + +export const DebugStateDiffResult = object({ + result: DebugStateDiff, + txHash: option(BYTES) +}) + + +export type DebugStateDiffResult = GetSrcType + + +const traceFrameBase = { + blockHash: option(BYTES), + transactionHash: option(BYTES), + traceAddress: array(NAT), + subtraces: NAT, + error: option(STRING), +} + + +export const TraceCreateAction = object({ + from: BYTES, + value: QTY, + gas: QTY, + init: BYTES +}) + + +export const TraceCreateResult = object({ + gasUsed: QTY, + code: BYTES, + address: BYTES +}) + + +export const TraceCallAction = object({ + from: BYTES, + to: BYTES, + value: QTY, + gas: QTY, + input: BYTES, + callType: oneOf({ + call: constant('call'), + callcode: constant('callcode'), + delegatecall: constant('delegatecall'), + staticcall: constant('staticcall') + }) +}) + + +export const TraceCallResult = object({ + gasUsed: QTY, + output: BYTES +}) + + +export const TraceSuicideAction = object({ + address: BYTES, + refundAddress: BYTES, + balance: QTY +}) + + +export const TraceRewardAction = object({ + author: BYTES, + value: QTY, + rewardType: oneOf({ + block: constant('block'), + uncle: constant('uncle'), + emptyStep: constant('emptyStep'), + external: constant('external') + }) +}) + + +export const TraceFrame = taggedUnion('type', { + create: object({ + ...traceFrameBase, + action: TraceCreateAction, + result: option(TraceCreateResult) + }), + call: object({ + ...traceFrameBase, + action: TraceCallAction, + result: option(TraceCallResult) + }), + suicide: object({ + ...traceFrameBase, + action: TraceSuicideAction + }), + reward: object({ + ...traceFrameBase, + action: TraceRewardAction + }) +}) + + +export type TraceFrame = GetSrcType + + +const TraceDiffObj = keyTaggedUnion({ + '+': BYTES, + '*': object({from: BYTES, to: BYTES}), + '-': BYTES +}) + + +const TraceDiff = oneOf({ + '= sign': constant('='), + 'diff object': TraceDiffObj +}) + + +export type TraceDiff = GetSrcType + + +export const TraceStateDiff = object({ + balance: TraceDiff, + code: TraceDiff, + nonce: TraceDiff, + storage: record(BYTES, TraceDiff) +}) + + +export type TraceStateDiff = GetSrcType + + +export const TraceTransactionReplay = object({ + transactionHash: option(BYTES), + trace: option(array(TraceFrame)), + stateDiff: option(record(BYTES, TraceStateDiff)), +}) + + +export type TraceTransactionReplay = GetSrcType + + +export const Block = object({ + height: NAT, + hash: BYTES, + block: GetBlock, + receipts: option(array(TransactionReceipt)), + logs: option(array(Log)), + traceReplays: option(array(TraceTransactionReplay)), + debugFrames: option(array(DebugFrameResult)), + debugStateDiffs: option(array(DebugStateDiffResult)) +}) + + +export type Block = GetSrcType diff --git a/evm/evm-dump/package.json b/evm/evm-dump/package.json index 44815e58e..0125bd0b8 100644 --- a/evm/evm-dump/package.json +++ b/evm/evm-dump/package.json @@ -22,6 +22,7 @@ "@subsquid/evm-data": "^0.0.0", "@subsquid/util-internal": "^3.0.0", "@subsquid/util-internal-ingest-tools": "^1.1.1", + "@subsquid/util-internal-validation": "^0.3.0", "@subsquid/util-internal-dump-cli": "^0.0.0" }, "devDependencies": { diff --git a/evm/evm-dump/src/dumper.ts b/evm/evm-dump/src/dumper.ts index 025e6252c..84fed87d2 100644 --- a/evm/evm-dump/src/dumper.ts +++ b/evm/evm-dump/src/dumper.ts @@ -1,8 +1,9 @@ -import {Block, DataRequest} from '@subsquid/evm-data/lib/rpc/rpc-data' -import {Rpc} from '@subsquid/evm-data/lib/rpc/rpc' +import {Block} from '@subsquid/evm-data/lib/schema' +import {Rpc, DataRequest} from '@subsquid/evm-data/lib/rpc' import {def} from '@subsquid/util-internal' import {coldIngest} from '@subsquid/util-internal-ingest-tools' -import {Command, Dumper, DumperOptions, positiveInt, Range} from '@subsquid/util-internal-dump-cli' +import {Command, Dumper, DumperOptions, positiveInt, Range, SplitRequest} from '@subsquid/util-internal-dump-cli' +import {DataValidationError} from '@subsquid/util-internal-validation' interface Options extends DumperOptions { @@ -45,7 +46,7 @@ export class EvmDumper extends Dumper { protected async* getBlocks(range: Range): AsyncIterable { let request: DataRequest = { - logs: true, + logs: !this.options().withReceipts, transactions: true, receipts: this.options().withReceipts, traces: this.options().withTraces, @@ -55,7 +56,7 @@ export class EvmDumper extends Dumper { } let batches = coldIngest({ getFinalizedHeight: () => this.getFinalizedHeight(), - getSplit: req => this.getDataSource().getColdSplit(req), + getSplit: req => this.getColdSplit(req), requests: [{range, request}], concurrency: Math.min(5, this.getDataSource().client.getConcurrency()), splitSize: 10, @@ -71,4 +72,15 @@ export class EvmDumper extends Dumper { let height = await this.getDataSource().getHeight() return Math.max(0, height - this.options().bestBlockOffset) } + + private async getColdSplit(req: SplitRequest): Promise { + let blocks = await this.getDataSource().getColdSplit(req) + for (let block of blocks) { + let err = Block.validate(block) + if (err) { + throw new DataValidationError(`server returned unexpected result: ${err.toString()}`) + } + } + return blocks as any + } } diff --git a/evm/evm-ingest/src/ingest.ts b/evm/evm-ingest/src/ingest.ts index eccb643a9..a6b66fed1 100644 --- a/evm/evm-ingest/src/ingest.ts +++ b/evm/evm-ingest/src/ingest.ts @@ -1,5 +1,5 @@ import {mapRpcBlock} from '@subsquid/evm-data/lib/normalization' -import {Block as RawBlock} from '@subsquid/evm-data/lib/rpc' +import {Block as RawBlock} from '@subsquid/evm-data/lib/schema' import {addErrorContext} from '@subsquid/util-internal' import {Command, Ingest, Range} from '@subsquid/util-internal-ingest-cli' import {toJSON} from '@subsquid/util-internal-json' diff --git a/evm/evm-processor/src/ds-archive/schema.ts b/evm/evm-processor/src/ds-archive/schema.ts index df4ca98e0..1253c8c84 100644 --- a/evm/evm-processor/src/ds-archive/schema.ts +++ b/evm/evm-processor/src/ds-archive/schema.ts @@ -1,29 +1,47 @@ import {weakMemo} from '@subsquid/util-internal' -import {array, BYTES, NAT, object, option, STRING, taggedUnion, withDefault} from '@subsquid/util-internal-validation' -import {FieldSelection} from '../interfaces/data' import { - getBlockHeaderProps, - getLogProps, - getTraceFrameValidator, - getTxProps, - getTxReceiptProps, - project -} from '../mapping/schema' + array, + BYTES, + NAT, + object, + option, + STRING, + taggedUnion, + withDefault, + withSentinel +} from '@subsquid/util-internal-validation' +import * as schema from '@subsquid/evm-data/lib/schema' +import {FieldSelection} from '../interfaces/data' +import {getTraceFrameValidator, project, getLogValidator} from '../mapping/schema' export const getBlockValidator = weakMemo((fields: FieldSelection) => { - let BlockHeader = object(getBlockHeaderProps(fields.block, true)) + let BlockHeader = object({ + ...project(fields.block, { + ...schema.GetBlock.props, + timestamp: withSentinel('BlockHeader.timestamp', 0, NAT), + size: withSentinel('BlockHeader.size', -1, NAT), + l1BlockNumber: withDefault(0, NAT), + }), + number: NAT, + hash: BYTES, + parentHash: BYTES, + }) let Transaction = object({ - hash: fields.transaction?.hash ? BYTES : undefined, - ...getTxProps(fields.transaction, true), - sighash: fields.transaction?.sighash ? withDefault('0x', BYTES) : undefined, - ...getTxReceiptProps(fields.transaction, true) + ...project(fields.transaction, { + ...schema.Transaction.props, + ...schema.TransactionReceipt.props, + nonce: withSentinel('Transaction.nonce', -1, NAT), + yParity: option(NAT), + chainId: option(NAT), + type: withSentinel('Receipt.type', -1, NAT), + status: withSentinel('Receipt.status', -1, NAT), + }), + transactionIndex: NAT, }) - let Log = object( - getLogProps(fields.log, true) - ) + let Log = getLogValidator(fields.log, true) let Trace = getTraceFrameValidator(fields.trace, true) diff --git a/evm/evm-processor/src/ds-rpc/mapping.ts b/evm/evm-processor/src/ds-rpc/mapping.ts index 64772779a..c5ad9c869 100644 --- a/evm/evm-processor/src/ds-rpc/mapping.ts +++ b/evm/evm-processor/src/ds-rpc/mapping.ts @@ -1,16 +1,15 @@ import {addErrorContext, assertNotNull, unexpectedCase} from '@subsquid/util-internal' import {cast, GetCastType} from '@subsquid/util-internal-validation' -import {GetPropsCast} from '@subsquid/util-internal-validation/lib/composite/object' import {Bytes, Bytes20, Bytes32} from '@subsquid/evm-data' import {Block as RpcBlock, DebugStateDiffResult, DebugStateMap, TraceDiff, TraceStateDiff} from '@subsquid/evm-data/lib/rpc' +import assert from 'assert' import { EvmTraceCallAction, EvmTraceCallResult, EvmTraceCreateAction, EvmTraceCreateResult, EvmTraceSuicideAction -} from '@subsquid/evm-data/lib/normalization' -import assert from 'assert' +} from '../interfaces/evm' import {FieldSelection} from '../interfaces/data' import { Block, @@ -29,7 +28,7 @@ import { Transaction } from '../mapping/entities' import {setUpRelations} from '../mapping/relations' -import {getLogProps, getTraceFrameValidator, isEmpty} from '../mapping/schema' +import {getLogValidator, getTraceFrameValidator, isEmpty} from '../mapping/schema' import {filterBlock} from './filter' import {MappingRequest} from './request' import {DebugFrame, getBlockValidator} from './schema' @@ -152,7 +151,7 @@ function tryMapBlock(rpcBlock: RpcBlock, req: MappingRequest): Block { } -function makeLog(blockHeader: BlockHeader, src: GetPropsCast>): Log { +function makeLog(blockHeader: BlockHeader, src: GetCastType>): Log { let {logIndex, transactionIndex, ...props} = src let log = new Log(blockHeader, logIndex, transactionIndex) Object.assign(log, props) diff --git a/evm/evm-processor/src/ds-rpc/request.ts b/evm/evm-processor/src/ds-rpc/request.ts index dffca5132..7cd4c3bb4 100644 --- a/evm/evm-processor/src/ds-rpc/request.ts +++ b/evm/evm-processor/src/ds-rpc/request.ts @@ -1,5 +1,5 @@ import {DataRequest as RpcDataRequest} from '@subsquid/evm-data/lib/rpc' -import {_EvmTx, _EvmTxReceipt} from '@subsquid/evm-data/lib/normalization' +import {_EvmTx, _EvmTxReceipt} from '../interfaces/evm' import {FieldSelection} from '../interfaces/data' import {DataRequest} from '../interfaces/data-request' diff --git a/evm/evm-processor/src/ds-rpc/schema.ts b/evm/evm-processor/src/ds-rpc/schema.ts index de117346a..181091eea 100644 --- a/evm/evm-processor/src/ds-rpc/schema.ts +++ b/evm/evm-processor/src/ds-rpc/schema.ts @@ -14,44 +14,45 @@ import { Validator, withDefault } from '@subsquid/util-internal-validation' -import {DebugStateDiffResult, TraceStateDiff, TransactionSchema} from '@subsquid/evm-data/lib/rpc' +import {DebugStateDiffResult, TraceStateDiff} from '@subsquid/evm-data/lib/rpc' import {Bytes, Bytes20} from '@subsquid/evm-data' import {FieldSelection} from '../interfaces/data' import { - getBlockHeaderProps, - getLogProps, + getLogValidator, getTraceFrameValidator, - getTxProps, - getTxReceiptProps, project } from '../mapping/schema' import {MappingRequest} from './request' +import * as schema from '@subsquid/evm-data/lib/schema' // Here we must be careful to include all fields, // that can potentially be used in item filters // (no matter what field selection is telling us to omit) export const getBlockValidator = weakMemo((req: MappingRequest) => { let Transaction = req.transactions - ? object(project(req.fields.transaction, TransactionSchema.props)) + ? object({ + ...project(req.fields.transaction, schema.Transaction.props), + hash: BYTES, + input: BYTES, + from: BYTES, + to: option(BYTES), + transactionIndex: SMALL_QTY, + }) : BYTES let GetBlock = object({ - ...getBlockHeaderProps(req.fields.block, false), + ...project(req.fields.block, schema.GetBlock.props), + number: SMALL_QTY, + hash: BYTES, + parentHash: BYTES, transactions: array(Transaction) }) - let Log = object({ - ...getLogProps( - {...req.fields.log, address: true, topics: true}, - false - ), - address: BYTES, - topics: array(BYTES) - }) + let Log = getLogValidator(req.fields.log, false) let Receipt = object({ + ...project(req.fields.transaction, schema.TransactionReceipt.props), transactionIndex: SMALL_QTY, - ...getTxReceiptProps(req.fields.transaction, false), logs: req.logList ? array(Log) : undefined }) diff --git a/evm/evm-processor/src/interfaces/data-request.ts b/evm/evm-processor/src/interfaces/data-request.ts index 5466798e5..88dc53755 100644 --- a/evm/evm-processor/src/interfaces/data-request.ts +++ b/evm/evm-processor/src/interfaces/data-request.ts @@ -1,5 +1,5 @@ import {Bytes, Bytes20, Bytes32} from '@subsquid/evm-data' -import {EvmStateDiff} from '@subsquid/evm-data/lib/normalization' +import {EvmStateDiff} from '../interfaces/evm' import {FieldSelection} from './data' diff --git a/evm/evm-processor/src/interfaces/data.ts b/evm/evm-processor/src/interfaces/data.ts index 1aaff214c..4b5a6a6a1 100644 --- a/evm/evm-processor/src/interfaces/data.ts +++ b/evm/evm-processor/src/interfaces/data.ts @@ -11,7 +11,7 @@ import { EvmTraceRewardAction, EvmTraceSuicideAction, EvmTransaction -} from '@subsquid/evm-data/lib/normalization' +} from './evm' type Simplify = { diff --git a/evm/evm-processor/src/interfaces/evm.ts b/evm/evm-processor/src/interfaces/evm.ts index 8c7ab91e1..e205fffc7 100644 --- a/evm/evm-processor/src/interfaces/evm.ts +++ b/evm/evm-processor/src/interfaces/evm.ts @@ -1,196 +1,196 @@ -// import {Bytes, Bytes20, Bytes32, Bytes8} from '@subsquid/evm-data' - - -// export interface EvmBlockHeader { -// height: number -// hash: Bytes32 -// parentHash: Bytes32 -// nonce: Bytes8 -// sha3Uncles: Bytes32 -// logsBloom: Bytes -// transactionsRoot: Bytes32 -// stateRoot: Bytes32 -// receiptsRoot: Bytes32 -// mixHash: Bytes -// miner: Bytes20 -// difficulty: bigint -// totalDifficulty: bigint -// extraData: Bytes -// size: bigint -// gasLimit: bigint -// gasUsed: bigint -// timestamp: number -// baseFeePerGas: bigint -// /** -// * This field is not supported by all currently deployed archives. -// * Requesting it may cause internal error. -// */ -// l1BlockNumber: number -// } - - -// export interface EvmTransaction extends _EvmTx, _EvmTxReceipt { -// transactionIndex: number -// sighash: Bytes -// } - - -// export interface _EvmTx { -// hash: Bytes32 -// from: Bytes20 -// to?: Bytes20 -// gas: bigint -// gasPrice: bigint -// maxFeePerGas?: bigint -// maxPriorityFeePerGas?: bigint -// input: Bytes -// nonce: number -// value: bigint -// v: bigint -// r: Bytes32 -// s: Bytes32 -// yParity?: number -// chainId?: number -// } - - -// export interface _EvmTxReceipt { -// gasUsed: bigint -// cumulativeGasUsed: bigint -// effectiveGasPrice: bigint -// contractAddress?: Bytes32 -// type: number -// status: number -// } - - -// export interface EvmLog { -// logIndex: number -// transactionIndex: number -// transactionHash: Bytes32 -// address: Bytes20 -// data: Bytes -// topics: Bytes32[] -// } - - -// export interface EvmTraceBase { -// transactionIndex: number -// traceAddress: number[] -// subtraces: number -// error: string | null -// revertReason?: string -// } - - -// export interface EvmTraceCreate extends EvmTraceBase { -// type: 'create' -// action: EvmTraceCreateAction -// result?: EvmTraceCreateResult -// } - - -// export interface EvmTraceCreateAction { -// from: Bytes20 -// value: bigint -// gas: bigint -// init: Bytes -// } - - -// export interface EvmTraceCreateResult { -// gasUsed: bigint -// code: Bytes -// address: Bytes20 -// } - - -// export interface EvmTraceCall extends EvmTraceBase { -// type: 'call' -// action: EvmTraceCallAction -// result?: EvmTraceCallResult -// } - - -// export interface EvmTraceCallAction { -// callType: string -// from: Bytes20 -// to: Bytes20 -// value?: bigint -// gas: bigint -// input: Bytes -// sighash: Bytes -// } +import {Bytes, Bytes20, Bytes32, Bytes8} from '@subsquid/evm-data' + + +export interface EvmBlockHeader { + height: number + hash: Bytes32 + parentHash: Bytes32 + nonce: Bytes8 + sha3Uncles: Bytes32 + logsBloom: Bytes + transactionsRoot: Bytes32 + stateRoot: Bytes32 + receiptsRoot: Bytes32 + mixHash: Bytes + miner: Bytes20 + difficulty: bigint + totalDifficulty: bigint + extraData: Bytes + size: bigint + gasLimit: bigint + gasUsed: bigint + timestamp: number + baseFeePerGas: bigint + /** + * This field is not supported by all currently deployed archives. + * Requesting it may cause internal error. + */ + l1BlockNumber: number +} + + +export interface EvmTransaction extends _EvmTx, _EvmTxReceipt { + transactionIndex: number + sighash: Bytes +} + + +export interface _EvmTx { + hash: Bytes32 + from: Bytes20 + to?: Bytes20 + gas: bigint + gasPrice: bigint + maxFeePerGas?: bigint + maxPriorityFeePerGas?: bigint + input: Bytes + nonce: number + value: bigint + v: bigint + r: Bytes32 + s: Bytes32 + yParity?: number + chainId?: number +} + + +export interface _EvmTxReceipt { + gasUsed: bigint + cumulativeGasUsed: bigint + effectiveGasPrice: bigint + contractAddress?: Bytes32 + type: number + status: number +} + + +export interface EvmLog { + logIndex: number + transactionIndex: number + transactionHash: Bytes32 + address: Bytes20 + data: Bytes + topics: Bytes32[] +} + + +export interface EvmTraceBase { + transactionIndex: number + traceAddress: number[] + subtraces: number + error: string | null + revertReason?: string +} + + +export interface EvmTraceCreate extends EvmTraceBase { + type: 'create' + action: EvmTraceCreateAction + result?: EvmTraceCreateResult +} + + +export interface EvmTraceCreateAction { + from: Bytes20 + value: bigint + gas: bigint + init: Bytes +} + + +export interface EvmTraceCreateResult { + gasUsed: bigint + code: Bytes + address: Bytes20 +} + + +export interface EvmTraceCall extends EvmTraceBase { + type: 'call' + action: EvmTraceCallAction + result?: EvmTraceCallResult +} + + +export interface EvmTraceCallAction { + callType: string + from: Bytes20 + to: Bytes20 + value?: bigint + gas: bigint + input: Bytes + sighash: Bytes +} -// export interface EvmTraceCallResult { -// gasUsed: bigint -// output: Bytes -// } +export interface EvmTraceCallResult { + gasUsed: bigint + output: Bytes +} -// export interface EvmTraceSuicide extends EvmTraceBase { -// type: 'suicide' -// action: EvmTraceSuicideAction -// } +export interface EvmTraceSuicide extends EvmTraceBase { + type: 'suicide' + action: EvmTraceSuicideAction +} -// export interface EvmTraceSuicideAction { -// address: Bytes20 -// refundAddress: Bytes20 -// balance: bigint -// } +export interface EvmTraceSuicideAction { + address: Bytes20 + refundAddress: Bytes20 + balance: bigint +} -// export interface EvmTraceReward extends EvmTraceBase { -// type: 'reward' -// action: EvmTraceRewardAction -// } +export interface EvmTraceReward extends EvmTraceBase { + type: 'reward' + action: EvmTraceRewardAction +} -// export interface EvmTraceRewardAction { -// author: Bytes20 -// value: bigint -// type: string -// } +export interface EvmTraceRewardAction { + author: Bytes20 + value: bigint + type: string +} -// export type EvmTrace = EvmTraceCreate | EvmTraceCall | EvmTraceSuicide | EvmTraceReward +export type EvmTrace = EvmTraceCreate | EvmTraceCall | EvmTraceSuicide | EvmTraceReward -// export interface EvmStateDiffBase { -// transactionIndex: number -// address: Bytes20 -// key: 'balance' | 'code' | 'nonce' | Bytes32 -// } +export interface EvmStateDiffBase { + transactionIndex: number + address: Bytes20 + key: 'balance' | 'code' | 'nonce' | Bytes32 +} -// export interface EvmStateDiffNoChange extends EvmStateDiffBase { -// kind: '=' -// prev?: null -// next?: null -// } +export interface EvmStateDiffNoChange extends EvmStateDiffBase { + kind: '=' + prev?: null + next?: null +} -// export interface EvmStateDiffAdd extends EvmStateDiffBase { -// kind: '+' -// prev?: null -// next: Bytes -// } +export interface EvmStateDiffAdd extends EvmStateDiffBase { + kind: '+' + prev?: null + next: Bytes +} -// export interface EvmStateDiffChange extends EvmStateDiffBase { -// kind: '*' -// prev: Bytes -// next: Bytes -// } +export interface EvmStateDiffChange extends EvmStateDiffBase { + kind: '*' + prev: Bytes + next: Bytes +} -// export interface EvmStateDiffDelete extends EvmStateDiffBase { -// kind: '-' -// prev: Bytes -// next?: null -// } +export interface EvmStateDiffDelete extends EvmStateDiffBase { + kind: '-' + prev: Bytes + next?: null +} -// export type EvmStateDiff = EvmStateDiffNoChange | EvmStateDiffAdd | EvmStateDiffChange | EvmStateDiffDelete +export type EvmStateDiff = EvmStateDiffNoChange | EvmStateDiffAdd | EvmStateDiffChange | EvmStateDiffDelete diff --git a/evm/evm-processor/src/mapping/entities.ts b/evm/evm-processor/src/mapping/entities.ts index f6e9ea259..74da57b3b 100644 --- a/evm/evm-processor/src/mapping/entities.ts +++ b/evm/evm-processor/src/mapping/entities.ts @@ -1,5 +1,6 @@ import {formatId} from '@subsquid/util-internal-processor-tools' import {Bytes, Bytes20, Bytes32, Bytes8} from '@subsquid/evm-data' +import assert from 'assert' import { EvmTraceCallAction, EvmTraceCallResult, @@ -7,8 +8,7 @@ import { EvmTraceCreateResult, EvmTraceRewardAction, EvmTraceSuicideAction -} from '@subsquid/evm-data/lib/normalization' -import assert from 'assert' +} from '../interfaces/evm' export class Block { diff --git a/evm/evm-processor/src/mapping/schema.ts b/evm/evm-processor/src/mapping/schema.ts index 993157764..6b01b6884 100644 --- a/evm/evm-processor/src/mapping/schema.ts +++ b/evm/evm-processor/src/mapping/schema.ts @@ -1,4 +1,3 @@ -import {FieldSelection} from '../interfaces/data' import { array, BYTES, @@ -9,90 +8,21 @@ import { SMALL_QTY, STRING, taggedUnion, - withDefault, - withSentinel + withDefault } from '@subsquid/util-internal-validation' +import * as schema from '@subsquid/evm-data/lib/schema' +import {FieldSelection} from '../interfaces/data' -export function getBlockHeaderProps(fields: FieldSelection['block'], forArchive: boolean) { - let natural = forArchive ? NAT : SMALL_QTY - return { - number: natural, - hash: BYTES, - parentHash: BYTES, - ...project(fields, { - nonce: withSentinel('BlockHeader.nonce', '0x', BYTES), - sha3Uncles: withSentinel('BlockHeader.sha3Uncles', '0x', BYTES), - logsBloom: withSentinel('BlockHeader.logsBloom', '0x', BYTES), - transactionsRoot: withSentinel('BlockHeader.transactionsRoot', '0x', BYTES), - stateRoot: withSentinel('BlockHeader.stateRoot', '0x', BYTES), - receiptsRoot: withSentinel('BlockHeader.receiptsRoot', '0x', BYTES), - mixHash: withSentinel('BlockHeader.mixHash', '0x', BYTES), - miner: withSentinel('BlockHeader.miner', '0x', BYTES), - difficulty: withSentinel('BlockHeader.difficulty', -1n, QTY), - totalDifficulty: withSentinel('BlockHeader.totalDifficulty', -1n, QTY), - extraData: withSentinel('BlockHeader.extraData', '0x', BYTES), - size: withSentinel('BlockHeader.size', -1, natural), - gasLimit: withSentinel('BlockHeader.gasLimit', -1n, QTY), - gasUsed: withSentinel('BlockHeader.gasUsed', -1n, QTY), - baseFeePerGas: withSentinel('BlockHeader.baseFeePerGas', -1n, QTY), - timestamp: withSentinel('BlockHeader.timestamp', 0, natural), - l1BlockNumber: withDefault(0, natural), - }) - } -} - - -export function getTxProps(fields: FieldSelection['transaction'], forArchive: boolean) { - let natural = forArchive ? NAT : SMALL_QTY - return { - transactionIndex: natural, - ...project(fields, { - hash: BYTES, - from: BYTES, - to: option(BYTES), - gas: withSentinel('Transaction.gas', -1n, QTY), - gasPrice: withSentinel('Transaction.gasPrice', -1n, QTY), - maxFeePerGas: option(QTY), - maxPriorityFeePerGas: option(QTY), - input: BYTES, - nonce: withSentinel('Transaction.nonce', -1, natural), - value: withSentinel('Transaction.value', -1n, QTY), - v: withSentinel('Transaction.v', -1n, QTY), - r: withSentinel('Transaction.r', '0x', BYTES), - s: withSentinel('Transaction.s', '0x', BYTES), - yParity: option(natural), - chainId: option(natural), - }) - } -} - - -export function getTxReceiptProps(fields: FieldSelection['transaction'], forArchive: boolean) { - let natural = forArchive ? NAT : SMALL_QTY - return project(fields, { - // gasUsed: withSentinel('Receipt.gasUsed', -1n, QTY), - // cumulativeGasUsed: withSentinel('Receipt.cumulativeGasUsed', -1n, QTY), - // effectiveGasPrice: withSentinel('Receipt.effectiveGasPrice', -1n, QTY), - // contractAddress: option(BYTES), - // type: withSentinel('Receipt.type', -1, natural), - // status: withSentinel('Receipt.status', -1, natural), - }) -} - - -export function getLogProps(fields: FieldSelection['log'], forArchive: boolean) { +export function getLogValidator(fields: FieldSelection['log'], forArchive: boolean) { let natural = forArchive ? NAT : SMALL_QTY - return { + return object({ + address: forArchive ? undefined : BYTES, + topics: forArchive ? undefined : array(BYTES), + ...project(fields, schema.Log.props), logIndex: natural, transactionIndex: natural, - ...project(fields, { - transactionHash: BYTES, - address: BYTES, - data: BYTES, - topics: array(BYTES) - }) - } + }) } @@ -228,6 +158,3 @@ export function isEmpty(obj: object): boolean { } return true } - - -export function assertAssignable(): void {} diff --git a/evm/evm-processor/src/mapping/selection.ts b/evm/evm-processor/src/mapping/selection.ts index a3005ec38..548cc9f81 100644 --- a/evm/evm-processor/src/mapping/selection.ts +++ b/evm/evm-processor/src/mapping/selection.ts @@ -50,12 +50,12 @@ export function getTxSelectionValidator() { s: FIELD, yParity: FIELD, chainId: FIELD, - // gasUsed: FIELD, - // cumulativeGasUsed: FIELD, - // effectiveGasPrice: FIELD, - // contractAddress: FIELD, - // type: FIELD, - // status: FIELD, + gasUsed: FIELD, + cumulativeGasUsed: FIELD, + effectiveGasPrice: FIELD, + contractAddress: FIELD, + type: FIELD, + status: FIELD, } return object(fields) } diff --git a/util/util-internal-validation/src/composite/object.ts b/util/util-internal-validation/src/composite/object.ts index f8b7f3897..0f789eaa7 100644 --- a/util/util-internal-validation/src/composite/object.ts +++ b/util/util-internal-validation/src/composite/object.ts @@ -13,7 +13,7 @@ export type GetPropsSrc = Simplify> -export class ObjectValidator> +export class ObjectValidator> implements Validator, GetPropsSrc> { constructor(public readonly props: Props) {} @@ -22,7 +22,7 @@ export class ObjectValidator> if (typeof object != 'object' || !object) return new ValidationFailure(object, `{value} is not an object`) let result: any = {} for (let key in this.props) { - let val = this.props[key].cast(object[key]) + let val = this.props[key]?.cast(object[key]) if (val === undefined) continue if (val instanceof ValidationFailure) { val.path.push(key) @@ -36,7 +36,7 @@ export class ObjectValidator> validate(object: any): ValidationFailure | undefined { if (typeof object != 'object' || !object) return new ValidationFailure(object, `{value} is not an object`) for (let key in this.props) { - let err = this.props[key].validate(object[key]) + let err = this.props[key]?.validate(object[key]) if (err) { err.path.push(key) return err diff --git a/util/util-internal-validation/src/dsl.ts b/util/util-internal-validation/src/dsl.ts index 39b552fc0..bec279427 100644 --- a/util/util-internal-validation/src/dsl.ts +++ b/util/util-internal-validation/src/dsl.ts @@ -15,17 +15,10 @@ import {DataValidationError, ValidationFailure} from './error' import {GetCastType, GetSrcType, Validator} from './interface' -export function object>>( +export function object | undefined>>( props: Props ): ObjectValidator { - let presentProps: Record> = {} - for (let key in props) { - let v = props[key] - if (v) { - presentProps[key] = v - } - } - return new ObjectValidator(presentProps) as ObjectValidator + return new ObjectValidator(props) } From 98a2bf363012e1ceaaf4dc4b63cf5981e2b88e3a Mon Sep 17 00:00:00 2001 From: tmcgroul Date: Sat, 16 Mar 2024 15:26:58 +0700 Subject: [PATCH 5/5] bring filtration back to object function --- evm/evm-data/src/rpc/rpc-data.ts | 2 +- .../src/composite/object.ts | 6 +++--- util/util-internal-validation/src/dsl.ts | 13 ++++++++++--- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/evm/evm-data/src/rpc/rpc-data.ts b/evm/evm-data/src/rpc/rpc-data.ts index 95441381b..4718a9eb2 100644 --- a/evm/evm-data/src/rpc/rpc-data.ts +++ b/evm/evm-data/src/rpc/rpc-data.ts @@ -65,7 +65,7 @@ export const Transaction = object({ blockHash: BYTES, transactionIndex: SMALL_QTY, hash: BYTES, - input: BYTES, + input: BYTES }) diff --git a/util/util-internal-validation/src/composite/object.ts b/util/util-internal-validation/src/composite/object.ts index 0f789eaa7..f8b7f3897 100644 --- a/util/util-internal-validation/src/composite/object.ts +++ b/util/util-internal-validation/src/composite/object.ts @@ -13,7 +13,7 @@ export type GetPropsSrc = Simplify> -export class ObjectValidator> +export class ObjectValidator> implements Validator, GetPropsSrc> { constructor(public readonly props: Props) {} @@ -22,7 +22,7 @@ export class ObjectValidator if (typeof object != 'object' || !object) return new ValidationFailure(object, `{value} is not an object`) let result: any = {} for (let key in this.props) { - let val = this.props[key]?.cast(object[key]) + let val = this.props[key].cast(object[key]) if (val === undefined) continue if (val instanceof ValidationFailure) { val.path.push(key) @@ -36,7 +36,7 @@ export class ObjectValidator validate(object: any): ValidationFailure | undefined { if (typeof object != 'object' || !object) return new ValidationFailure(object, `{value} is not an object`) for (let key in this.props) { - let err = this.props[key]?.validate(object[key]) + let err = this.props[key].validate(object[key]) if (err) { err.path.push(key) return err diff --git a/util/util-internal-validation/src/dsl.ts b/util/util-internal-validation/src/dsl.ts index bec279427..8c2f4b406 100644 --- a/util/util-internal-validation/src/dsl.ts +++ b/util/util-internal-validation/src/dsl.ts @@ -3,7 +3,7 @@ import {ConstantValidator} from './composite/constant' import {Default} from './composite/default' import {GetKeyTaggedUnionCast, GetKeyTaggedUnionSrc, KeyTaggedUnionValidator} from './composite/key-tagged-union' import {NullableValidator} from './composite/nullable' -import {GetPropsCast, GetPropsSrc, ObjectValidator} from './composite/object' +import {ObjectValidator} from './composite/object' import {GetOneOfCast, GetOneOfSrc, OneOfValidator} from './composite/one-of' import {OptionValidator} from './composite/option' import {RecordValidator} from './composite/record' @@ -17,8 +17,15 @@ import {GetCastType, GetSrcType, Validator} from './interface' export function object | undefined>>( props: Props -): ObjectValidator { - return new ObjectValidator(props) +): ObjectValidator<{[K in keyof Props]: NonNullable}> { + let presentProps: Record> = {} + for (let key in props) { + let v = props[key] + if (v) { + presentProps[key] = v + } + } + return new ObjectValidator(presentProps) as any }