diff --git a/common/changes/@subsquid/evm-processor/optional_validation_2024-10-09-14-50.json b/common/changes/@subsquid/evm-processor/optional_validation_2024-10-09-14-50.json new file mode 100644 index 00000000..24467b99 --- /dev/null +++ b/common/changes/@subsquid/evm-processor/optional_validation_2024-10-09-14-50.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@subsquid/evm-processor", + "comment": "Introduce flags to make the RPC data validation checks optional", + "type": "minor" + } + ], + "packageName": "@subsquid/evm-processor" +} \ No newline at end of file diff --git a/evm/evm-processor/src/ds-rpc/client.ts b/evm/evm-processor/src/ds-rpc/client.ts index 64d5092d..4903c710 100644 --- a/evm/evm-processor/src/ds-rpc/client.ts +++ b/evm/evm-processor/src/ds-rpc/client.ts @@ -29,7 +29,7 @@ import {DataRequest} from '../interfaces/data-request' import {Block} from '../mapping/entities' import {mapBlock} from './mapping' import {MappingRequest, toMappingRequest} from './request' -import {Rpc} from './rpc' +import {Rpc, ValidationFlags} from './rpc' const NO_REQUEST = toMappingRequest() @@ -44,9 +44,9 @@ export interface EvmRpcDataSourceOptions { useDebugApiForStateDiffs?: boolean debugTraceTimeout?: string log?: Logger + validationFlags?: ValidationFlags } - export class EvmRpcDataSource implements HotDataSource { private rpc: Rpc private finalityConfirmation: number @@ -56,10 +56,10 @@ export class EvmRpcDataSource implements HotDataSource { private useDebugApiForStateDiffs?: boolean private debugTraceTimeout?: string private log?: Logger - + constructor(options: EvmRpcDataSourceOptions) { this.log = options.log - this.rpc = new Rpc(options.rpc, this.log) + this.rpc = new Rpc(options.rpc, this.log, options.validationFlags) this.finalityConfirmation = options.finalityConfirmation this.headPollInterval = options.headPollInterval || 5_000 this.newHeadTimeout = options.newHeadTimeout || 0 diff --git a/evm/evm-processor/src/ds-rpc/rpc.ts b/evm/evm-processor/src/ds-rpc/rpc.ts index 8c38e26f..5e0c4364 100644 --- a/evm/evm-processor/src/ds-rpc/rpc.ts +++ b/evm/evm-processor/src/ds-rpc/rpc.ts @@ -39,22 +39,50 @@ function getResultValidator(validator: V): (result: unknown } } +export interface ValidationFlags { + /** + * Checks the logs list is non-empty if logsBloom is non-zero + */ + disableLogsBloomCheck?: boolean + /** + * Checks the tx count matches the number tx receipts + */ + disableTxReceiptsNumberCheck?:boolean, + /** + * Checks if the are no traces for a non-empty block + */ + disableMissingTracesCheck?:boolean + /** + * Checks the block hash matches the trace blockHash field + */ + disableTraceBlockHashCheck?:boolean +} export class Rpc { private props: RpcProps - + constructor( public readonly client: RpcClient, private log?: Logger, + private validationFlags: ValidationFlags = {}, private genesisHeight: number = 0, private priority: number = 0, - props?: RpcProps + props?: RpcProps, ) { this.props = props || new RpcProps(this.client, this.genesisHeight) + if (this.validationFlags.disableLogsBloomCheck) { + log?.warn(`Log bloom check is disabled`) + } + if (this.validationFlags.disableMissingTracesCheck) { + log?.warn(`Missing traces check is disabled`) + } + if (this.validationFlags.disableTxReceiptsNumberCheck) { + log?.warn(`Tx recipt number check is disabled`) + } } withPriority(priority: number): Rpc { - return new Rpc(this.client, this.log, this.genesisHeight, priority, this.props) + return new Rpc(this.client, this.log, this.validationFlags, this.genesisHeight, priority, this.props) } call(method: string, params?: any[], options?: CallOptions): Promise { @@ -242,12 +270,13 @@ export class Rpc { for (let block of blocks) { let logs = logsByBlock.get(block.hash) || [] - if (logs.length == 0 && block.block.logsBloom !== NO_LOGS_BLOOM) { + block.logs = logs + + const isCheckDisabled = this.validationFlags?.disableLogsBloomCheck ?? false + if (!isCheckDisabled && (logs.length === 0 && block.block.logsBloom !== NO_LOGS_BLOOM)) { block._isInvalid = true block._errorMessage = 'got 0 log records from eth_getLogs, but logs bloom is not empty' - } else { - block.logs = logs - } + } } } @@ -348,18 +377,28 @@ export class Rpc { if (receipts == null) { block._isInvalid = true block._errorMessage = `${method} returned null` - } else if (block.block.transactions.length === receipts.length) { - for (let receipt of receipts) { - if (receipt.blockHash !== block.hash) { - block._isInvalid = true - block._errorMessage = `${method} returned receipts for a different block` - } + continue + } + + block.receipts = receipts + + //block hash check + for (let receipt of receipts) { + const disableTraceBlockHashCheck = this.validationFlags?.disableTraceBlockHashCheck ?? false + if (!disableTraceBlockHashCheck && (receipt.blockHash !== block.hash)) { + // for the hash mismatch, fail anyway + block._isInvalid = true + block._errorMessage = `${method} returned receipts for a different block` } - block.receipts = receipts - } else { + } + + // count match check + const disableTxReceiptsNumberCheck = this.validationFlags?.disableTxReceiptsNumberCheck ?? false + if (!disableTxReceiptsNumberCheck && (block.block.transactions.length !== receipts.length)) { block._isInvalid = true block._errorMessage = `got invalid number of receipts from ${method}` - } + } + } } @@ -385,9 +424,10 @@ export class Rpc { for (let block of blocks) { let rs = receiptsByBlock.get(block.hash) || [] - if (rs.length === block.block.transactions.length) { - block.receipts = rs - } else { + block.receipts = rs + + const isCheckDisabled = this.validationFlags?.disableTxReceiptsNumberCheck ?? false + if (!isCheckDisabled && (rs.length !== block.block.transactions.length)) { block._isInvalid = true block._errorMessage = 'failed to get receipts for all transactions' } @@ -458,22 +498,28 @@ export class Rpc { validateResult: getResultValidator(array(TraceFrame)) }) + for (let i = 0; i < blocks.length; i++) { let block = blocks[i] let frames = results[i] + if (frames.length == 0) { - if (block.block.transactions.length > 0) { + const isCheckDisabled = this.validationFlags?.disableMissingTracesCheck ?? false + + if (!isCheckDisabled && (block.block.transactions.length > 0)) { block._isInvalid = true block._errorMessage = 'missing traces for some transactions' } - } else { - for (let frame of frames) { - if (frame.blockHash !== block.hash) { + continue + } + + for (let frame of frames) { + if (frame.blockHash !== block.hash) { block._isInvalid = true block._errorMessage = 'trace_block returned a trace of a different block' break - } } + if (!block._isInvalid) { block.traceReplays = [] let byTx = groupBy(frames, f => f.transactionHash) diff --git a/evm/evm-processor/src/processor.ts b/evm/evm-processor/src/processor.ts index fa27e7f4..0aabea85 100644 --- a/evm/evm-processor/src/processor.ts +++ b/evm/evm-processor/src/processor.ts @@ -80,8 +80,31 @@ export interface RpcDataIngestionSettings { * Disable RPC data ingestion entirely */ disabled?: boolean + + /** + * Flags to switch off the data consistency checks + */ + validationFlags?: RpcValidationFlags } +export interface RpcValidationFlags { + /** + * Checks the logs list is non-empty if logsBloom is non-zero + */ + disableLogsBloomCheck?: boolean + /** + * Checks the tx count matches the number tx receipts + */ + disableTxReceiptsNumberCheck?:boolean, + /** + * Checks if the are no traces for a non-empty block + */ + disableMissingTracesCheck?:boolean + /** + * Checks the block hash matches the trace blockHash field + */ + disableTraceBlockHashCheck?:boolean +} export interface GatewaySettings { /** @@ -466,6 +489,7 @@ export class EvmBatchProcessor { debugTraceTimeout: this.rpcIngestSettings?.debugTraceTimeout, headPollInterval: this.rpcIngestSettings?.headPollInterval, newHeadTimeout: this.rpcIngestSettings?.newHeadTimeout, + validationFlags: this.rpcIngestSettings?.validationFlags, log: this.getLogger().child('rpc', {rpcUrl: this.getChainRpcClient().url}) }) }