Optional validation flags

common/changes/@subsquid/evm-processor/optional_validation_2024-10-09-14-50.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@subsquid/evm-processor",
+      "comment": "Introduce flags to make the RPC data validation checks optional",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@subsquid/evm-processor"
+}

evm/evm-processor/src/ds-rpc/client.ts

@@ -29,7 +29,7 @@ import {DataRequest} from '../interfaces/data-request'
 import {Block} from '../mapping/entities'
 import {mapBlock} from './mapping'
 import {MappingRequest, toMappingRequest} from './request'
-import {Rpc} from './rpc'
+import {Rpc, ValidationFlags} from './rpc'
 
 
 const NO_REQUEST = toMappingRequest()
@@ -44,9 +44,9 @@ export interface EvmRpcDataSourceOptions {
     useDebugApiForStateDiffs?: boolean
     debugTraceTimeout?: string
     log?: Logger
+    validationFlags?: ValidationFlags
 }
 
-
 export class EvmRpcDataSource implements HotDataSource<Block, DataRequest> {
     private rpc: Rpc
     private finalityConfirmation: number
@@ -56,10 +56,10 @@ export class EvmRpcDataSource implements HotDataSource<Block, DataRequest> {
     private useDebugApiForStateDiffs?: boolean
     private debugTraceTimeout?: string
     private log?: Logger
-
+    
     constructor(options: EvmRpcDataSourceOptions) {
         this.log = options.log
-        this.rpc = new Rpc(options.rpc, this.log)
+        this.rpc = new Rpc(options.rpc, this.log, options.validationFlags)
         this.finalityConfirmation = options.finalityConfirmation
         this.headPollInterval = options.headPollInterval || 5_000
         this.newHeadTimeout = options.newHeadTimeout || 0

evm/evm-processor/src/ds-rpc/rpc.ts

@@ -39,22 +39,50 @@ function getResultValidator<V extends Validator>(validator: V): (result: unknown
     }
 }
 
+export interface ValidationFlags {
+    /**
+     * Checks the logs list is non-empty if logsBloom is non-zero
+     */
+    disableLogsBloomCheck?: boolean 
+    /**
+     * Checks the tx count matches the number tx receipts
+     */
+    disableTxReceiptsNumberCheck?:boolean,
+    /**
+     * Checks if the are no traces for a non-empty block
+     */
+    disableMissingTracesCheck?:boolean
+    /**
+     * Checks the block hash matches the trace blockHash field
+     */
+    disableTraceBlockHashCheck?:boolean
+}
 
 export class Rpc {
     private props: RpcProps
-
+    
     constructor(
         public readonly client: RpcClient,
         private log?: Logger,
+        private validationFlags: ValidationFlags = {},
         private genesisHeight: number = 0,
         private priority: number = 0,
-        props?: RpcProps
+        props?: RpcProps,
     ) {
         this.props = props || new RpcProps(this.client, this.genesisHeight)
+        if (this.validationFlags.disableLogsBloomCheck) {
+            log?.warn(`Log bloom check is disabled`)
+        }
+        if (this.validationFlags.disableMissingTracesCheck) {
+            log?.warn(`Missing traces check is disabled`)
+        }
+        if (this.validationFlags.disableTxReceiptsNumberCheck) {
+            log?.warn(`Tx recipt number check is disabled`)
+        }
     }
 
     withPriority(priority: number): Rpc {
-        return new Rpc(this.client, this.log, this.genesisHeight, priority, this.props)
+        return new Rpc(this.client, this.log, this.validationFlags, this.genesisHeight, priority, this.props)
     }
 
     call<T=any>(method: string, params?: any[], options?: CallOptions<T>): Promise<T> {
@@ -242,12 +270,13 @@ export class Rpc {
 
         for (let block of blocks) {
             let logs = logsByBlock.get(block.hash) || []
-            if (logs.length == 0 && block.block.logsBloom !== NO_LOGS_BLOOM) {
+            block.logs = logs
+
+            const isCheckDisabled = this.validationFlags?.disableLogsBloomCheck ?? false
+            if (!isCheckDisabled && (logs.length === 0 && block.block.logsBloom !== NO_LOGS_BLOOM)) {
                 block._isInvalid = true
                 block._errorMessage = 'got 0 log records from eth_getLogs, but logs bloom is not empty'
-            } else {
-                block.logs = logs
-            }
+            } 
         }
     }
 
@@ -348,18 +377,28 @@ export class Rpc {
             if (receipts == null) {
                 block._isInvalid = true
                 block._errorMessage = `${method} returned null`
-            } else if (block.block.transactions.length === receipts.length) {
-                for (let receipt of receipts) {
-                    if (receipt.blockHash !== block.hash) {
-                        block._isInvalid = true
-                        block._errorMessage = `${method} returned receipts for a different block`
-                    }
+                continue
+            } 
+
+            block.receipts = receipts
+
+            //block hash check
+            for (let receipt of receipts) {
+                const disableTraceBlockHashCheck = this.validationFlags?.disableTraceBlockHashCheck ?? false       
+                if (!disableTraceBlockHashCheck && (receipt.blockHash !== block.hash)) {
+                    // for the hash mismatch, fail anyway
+                    block._isInvalid = true
+                    block._errorMessage = `${method} returned receipts for a different block`
                 }
-                block.receipts = receipts
-            } else {
+            }
+
+            // count match check
+            const disableTxReceiptsNumberCheck = this.validationFlags?.disableTxReceiptsNumberCheck ?? false
+            if (!disableTxReceiptsNumberCheck && (block.block.transactions.length !== receipts.length)) {
                 block._isInvalid = true
                 block._errorMessage = `got invalid number of receipts from ${method}`
-            }
+            } 
+
         }
     }
 
@@ -385,9 +424,10 @@ export class Rpc {
 
         for (let block of blocks) {
             let rs = receiptsByBlock.get(block.hash) || []
-            if (rs.length === block.block.transactions.length) {
-                block.receipts = rs
-            } else {
+            block.receipts = rs
+
+            const isCheckDisabled = this.validationFlags?.disableTxReceiptsNumberCheck ?? false       
+            if (!isCheckDisabled && (rs.length !== block.block.transactions.length)) {
                 block._isInvalid = true
                 block._errorMessage = 'failed to get receipts for all transactions'
             }
@@ -458,22 +498,28 @@ export class Rpc {
             validateResult: getResultValidator(array(TraceFrame))
         })
 
+
         for (let i = 0; i < blocks.length; i++) {
             let block = blocks[i]
             let frames = results[i]
+
             if (frames.length == 0) {
-                if (block.block.transactions.length > 0) {
+                const isCheckDisabled = this.validationFlags?.disableMissingTracesCheck ?? false
+
+                if (!isCheckDisabled && (block.block.transactions.length > 0)) {
                     block._isInvalid = true
                     block._errorMessage = 'missing traces for some transactions'
                 }
-            } else {
-                for (let frame of frames) {
-                    if (frame.blockHash !== block.hash) {
+                continue
+            }  
+
+            for (let frame of frames) {
+                if (frame.blockHash !== block.hash) {
                         block._isInvalid = true
                         block._errorMessage = 'trace_block returned a trace of a different block'
                         break
-                    }
                 }
+
                 if (!block._isInvalid) {
                     block.traceReplays = []
                     let byTx = groupBy(frames, f => f.transactionHash)

evm/evm-processor/src/processor.ts

@@ -80,8 +80,31 @@ export interface RpcDataIngestionSettings {
      * Disable RPC data ingestion entirely
      */
     disabled?: boolean
+
+    /**
+     * Flags to switch off the data consistency checks
+     */
+    validationFlags?: RpcValidationFlags 
 }
 
+export interface RpcValidationFlags {
+    /**
+     * Checks the logs list is non-empty if logsBloom is non-zero
+     */
+    disableLogsBloomCheck?: boolean 
+    /**
+     * Checks the tx count matches the number tx receipts
+     */
+    disableTxReceiptsNumberCheck?:boolean,
+    /**
+     * Checks if the are no traces for a non-empty block
+     */
+    disableMissingTracesCheck?:boolean
+    /**
+     * Checks the block hash matches the trace blockHash field
+     */
+    disableTraceBlockHashCheck?:boolean
+}
 
 export interface GatewaySettings {
     /**
@@ -466,6 +489,7 @@ export class EvmBatchProcessor<F extends FieldSelection = {}> {
             debugTraceTimeout: this.rpcIngestSettings?.debugTraceTimeout,
             headPollInterval: this.rpcIngestSettings?.headPollInterval,
             newHeadTimeout: this.rpcIngestSettings?.newHeadTimeout,
+            validationFlags: this.rpcIngestSettings?.validationFlags,
             log: this.getLogger().child('rpc', {rpcUrl: this.getChainRpcClient().url})
         })
     }