add field selection schema validation

common/changes/@subsquid/evm-processor/field-selection-schema_2024-02-21-13-33.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@subsquid/evm-processor",
+      "comment": "add field selection schema validation",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@subsquid/evm-processor"
+}

common/changes/@subsquid/substrate-processor/field-selection-schema_2024-02-21-13-33.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@subsquid/substrate-processor",
+      "comment": "add field selection schema validation",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@subsquid/substrate-processor"
+}

common/changes/@subsquid/util-internal-validation/field-selection-schema_2024-02-21-13-33.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@subsquid/util-internal-validation",
+      "comment": "add `BOOLEAN` primitive",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@subsquid/util-internal-validation"
+}

evm/evm-processor/src/mapping/selection.ts

@@ -0,0 +1,125 @@
+import {FieldSelection} from '../interfaces/data'
+import {object, option, BOOLEAN} from '@subsquid/util-internal-validation'
+
+
+type GetFieldSelectionSchema<T> = {[K in keyof T]-?: typeof FIELD}
+
+
+const FIELD = option(BOOLEAN)
+
+
+export function getBlockHeaderSelectionValidator() {
+    let fields: GetFieldSelectionSchema<FieldSelection['block']> = {
+        nonce: FIELD,
+        sha3Uncles: FIELD,
+        logsBloom: FIELD,
+        transactionsRoot: FIELD,
+        stateRoot: FIELD,
+        receiptsRoot: FIELD,
+        mixHash: FIELD,
+        miner: FIELD,
+        difficulty: FIELD,
+        totalDifficulty: FIELD,
+        extraData: FIELD,
+        size: FIELD,
+        gasLimit: FIELD,
+        gasUsed: FIELD,
+        baseFeePerGas: FIELD,
+        timestamp: FIELD,
+        l1BlockNumber: FIELD,
+    }
+    return object(fields)
+}
+
+
+export function getTxSelectionValidator() {
+    let fields: GetFieldSelectionSchema<FieldSelection['transaction']> = {
+        hash: FIELD,
+        from: FIELD,
+        to: FIELD,
+        gas: FIELD,
+        gasPrice: FIELD,
+        maxFeePerGas: FIELD,
+        maxPriorityFeePerGas: FIELD,
+        sighash: FIELD,
+        input: FIELD,
+        nonce: FIELD,
+        value: FIELD,
+        v: FIELD,
+        r: FIELD,
+        s: FIELD,
+        yParity: FIELD,
+        chainId: FIELD,
+        gasUsed: FIELD,
+        cumulativeGasUsed: FIELD,
+        effectiveGasPrice: FIELD,
+        contractAddress: FIELD,
+        type: FIELD,
+        status: FIELD,
+    }
+    return object(fields)
+}
+
+
+export function getLogSelectionValidator() {
+    let fields: GetFieldSelectionSchema<FieldSelection['log']> = {
+        transactionHash: FIELD,
+        address: FIELD,
+        data: FIELD,
+        topics: FIELD,
+    }
+    return object(fields)
+}
+
+
+export function getTraceSelectionValidator() {
+    let fields: GetFieldSelectionSchema<FieldSelection['trace']> = {
+        callCallType: FIELD,
+        callFrom: FIELD,
+        callGas: FIELD,
+        callInput: FIELD,
+        callResultGasUsed: FIELD,
+        callResultOutput: FIELD,
+        callSighash: FIELD,
+        callTo: FIELD,
+        callValue: FIELD,
+        createFrom: FIELD,
+        createGas: FIELD,
+        createInit: FIELD,
+        createResultAddress: FIELD,
+        createResultCode: FIELD,
+        createResultGasUsed: FIELD,
+        createValue: FIELD,
+        error: FIELD,
+        revertReason: FIELD,
+        rewardAuthor: FIELD,
+        rewardType: FIELD,
+        rewardValue: FIELD,
+        subtraces: FIELD,
+        suicideAddress: FIELD,
+        suicideBalance: FIELD,
+        suicideRefundAddress: FIELD,
+    }
+    return object(fields)
+}
+
+
+export function getStateDiffSelectionValidator() {
+    let fields: GetFieldSelectionSchema<FieldSelection['stateDiff']> = {
+        kind: FIELD,
+        next: FIELD,
+        prev: FIELD,
+    }
+    return object(fields)
+}
+
+
+export function getFieldSelectionValidator() {
+    return object({
+        block: option(getBlockHeaderSelectionValidator()),
+        log: option(getLogSelectionValidator()),
+        transaction: option(getTxSelectionValidator()),
+        trace: option(getTraceSelectionValidator()),
+        stateDiff: option(getStateDiffSelectionValidator()),
+    })
+}

evm/evm-processor/src/processor.ts

@@ -5,12 +5,14 @@ import {assertNotNull, def, runProgram} from '@subsquid/util-internal'
 import {ArchiveClient} from '@subsquid/util-internal-archive-client'
 import {Database, getOrGenerateSquidId, PrometheusServer, Runner} from '@subsquid/util-internal-processor-tools'
 import {applyRangeBound, mergeRangeRequests, Range, RangeRequest} from '@subsquid/util-internal-range'
+import {cast} from '@subsquid/util-internal-validation'
 import assert from 'assert'
 import {EvmArchive} from './ds-archive/client'
 import {EvmRpcDataSource} from './ds-rpc/client'
 import {Chain} from './interfaces/chain'
 import {BlockData, DEFAULT_FIELDS, FieldSelection} from './interfaces/data'
 import {DataRequest, LogRequest, StateDiffRequest, TraceRequest, TransactionRequest} from './interfaces/data-request'
+import {getFieldSelectionValidator} from './mapping/selection'
 
 
 export interface RpcEndpointSettings {
@@ -314,7 +316,8 @@ export class EvmBatchProcessor<F extends FieldSelection = {}> {
      */
     setFields<T extends FieldSelection>(fields: T): EvmBatchProcessor<T> {
         this.assertNotRunning()
-        this.fields = fields
+        let validator = getFieldSelectionValidator()
+        this.fields = cast(validator, fields)
         return this as any
     }
 

substrate/substrate-processor/package.json

@@ -27,7 +27,8 @@
     "@subsquid/util-internal-ingest-tools": "^1.1.0",
     "@subsquid/util-internal-json": "^1.2.2",
     "@subsquid/util-internal-processor-tools": "^4.0.1",
-    "@subsquid/util-internal-range": "^0.1.0"
+    "@subsquid/util-internal-range": "^0.1.0",
+    "@subsquid/util-internal-validation": "~0.2.0"
   },
   "peerDependencies": {
     "@subsquid/substrate-runtime": "^1.0.3"

substrate/substrate-processor/src/processor.ts

@@ -15,6 +15,7 @@ import {assertNotNull, def, runProgram} from '@subsquid/util-internal'
 import {ArchiveClient} from '@subsquid/util-internal-archive-client'
 import {Batch, Database, getOrGenerateSquidId, PrometheusServer, Runner} from '@subsquid/util-internal-processor-tools'
 import {applyRangeBound, mergeRangeRequests, Range, RangeRequest} from '@subsquid/util-internal-range'
+import {cast} from '@subsquid/util-internal-validation'
 import assert from 'assert'
 import {Chain} from './chain'
 import {SubstrateArchive} from './ds-archive'
@@ -30,6 +31,7 @@ import {
     GearMessageQueuedRequest,
     GearUserMessageSentRequest
 } from './interfaces/data-request'
+import {getFieldSelectionValidator} from './selection'
 
 
 export interface RpcEndpointSettings {
@@ -284,7 +286,8 @@ export class SubstrateBatchProcessor<F extends FieldSelection = {}> {
      */
     setFields<T extends FieldSelection>(fields: T): SubstrateBatchProcessor<T> {
         this.assertNotRunning()
-        this.fields = fields
+        let validator = getFieldSelectionValidator()
+        this.fields = cast(validator, fields)
         return this as any
     }
 

substrate/substrate-processor/src/selection.ts

@@ -0,0 +1,66 @@
+import {FieldSelection} from './interfaces/data'
+import {object, option, BOOLEAN} from '@subsquid/util-internal-validation'
+
+
+type GetFieldSelectionSchema<T> = {[K in keyof T]-?: typeof FIELD}
+
+
+const FIELD = option(BOOLEAN)
+
+
+export function getBlockHeaderSelectionValidator() {
+    let fields: GetFieldSelectionSchema<FieldSelection['block']> = {
+        digest: FIELD,
+        extrinsicsRoot: FIELD,
+        stateRoot: FIELD,
+        timestamp: FIELD,
+        validator: FIELD,
+    }
+    return object(fields)
+}
+
+
+export function getExtrinsicSelectionValidator() {
+    let fields: GetFieldSelectionSchema<FieldSelection['extrinsic']> = {
+        hash: FIELD,
+        error: FIELD,
+        fee: FIELD,
+        signature: FIELD,
+        success: FIELD,
+        tip: FIELD,
+        version: FIELD,
+    }
+    return object(fields)
+}
+
+
+export function getEventSelectionValidator() {
+    let fields: GetFieldSelectionSchema<FieldSelection['event']> = {
+        args: FIELD,
+        name: FIELD,
+        phase: FIELD,
+    }
+    return object(fields)
+}
+
+
+export function getCallSelectionValidator() {
+    let fields: GetFieldSelectionSchema<FieldSelection['call']> = {
+        args: FIELD,
+        error: FIELD,
+        name: FIELD,
+        origin: FIELD,
+        success: FIELD
+    }
+    return object(fields)
+}
+
+
+export function getFieldSelectionValidator() {
+    return object({
+        block: option(getBlockHeaderSelectionValidator()),
+        extrinsic: option(getExtrinsicSelectionValidator()),
+        call: option(getCallSelectionValidator()),
+        event: option(getEventSelectionValidator()),
+    })
+}

test/erc20-transfers/src/processor.ts

@@ -14,7 +14,7 @@ const processor = new EvmBatchProcessor()
     .setBlockRange({from: 153000000})
     .setFields({
         block: {size: true},
-        log: {transactionHash: true}
+        log: {transactionHash: true, foo: true}
     })
     .addLog({
         address: [CONTRACT],

util/util-internal-validation/src/primitives.ts

@@ -140,3 +140,16 @@ export const BYTES: Validator<Bytes> = {
         return '0x'
     }
 }
+
+export const BOOLEAN: Validator<boolean> = {
+    cast(value: unknown): boolean | ValidationFailure {
+        return this.validate(value) || value as boolean
+    },
+    validate(value: unknown): ValidationFailure | undefined {
+        if (typeof value === 'boolean') return
+        return new ValidationFailure(value, `{value} is not a boolean`)
+    },
+    phantom(): boolean {
+        return false
+    }
+}